diff --git a/.gitignore b/.gitignore index 7a68b30..fb2291e 100644 --- a/.gitignore +++ b/.gitignore @@ -190,7 +190,7 @@ cython_debug/ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore # and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder -# .vscode/ +.vscode/ # Ruff stuff: .ruff_cache/ @@ -223,7 +223,11 @@ benchmark_comparison.txt *.pwd.yaml */osw_files/* +# Node (src/oold/ui/vue) +node_modules/ + # Local CLAUDE.md AGENTS.md +.claude/ .ign diff --git a/README.md b/README.md index 91b2c0f..b4a119e 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,24 @@ f = model.Foo( Thanks to the resolver mechanism, these IRIs turn into fully-fledged objects as soon as you need them. -More details see [example code](./tests/test_oold.py) +Writing a model by hand, declare links with `Link[T]` / `LinkList[T]`. This is the recommended notation: it is the only one a type checker reads correctly in both directions - the resolved object you get back, and the object, IRI or JSON object you may assign: + +```python +from oold.model import Link, LinkedBaseModel, LinkList, OoldField + +class Person(LinkedBaseModel): + id: str + name: str | None = None + employer: Link["Organization | None"] = OoldField() + knows: LinkList["Person"] = OoldField() + +alice = Person(id="ex:alice", knows=["ex:bob", {"id": "ex:carol"}]) +alice.knows[0].name # a Person, resolved on access +``` + +Existing declarations keep working unchanged, including the `range` form code generation emits. + +More details see [example code](./tests/test_oold.py) and [Object Graph Mapping](./docs/how-to/object-graph-mapping.md). ### RDF-Export diff --git a/docs/design/downstream-migration.md b/docs/design/downstream-migration.md new file mode 100644 index 0000000..83a444b --- /dev/null +++ b/docs/design/downstream-migration.md @@ -0,0 +1,291 @@ +# Downstream API surface and which patterns become obsolete + +Companion to [graph-object-binding.md](graph-object-binding.md), tracked in +[oold-python#107]. + +Downstream inherits its API from `oold.model.LinkedBaseModel` through +`opensemantic.OswBaseModel`: + +``` +opensemantic.OswBaseModel -> oold.model.LinkedBaseModel -> pydantic.BaseModel +opensemantic.v1.OswBaseModel -> oold.model.v1.LinkedBaseModel -> pydantic.v1.BaseModel +``` + +`to_json`, `to_jsonld`, `from_json` and `from_jsonld` are **inherited from +`LinkedBaseModel`**, not defined by `OswBaseModel`, so replacing the binding +changes them for every consumer. + +## Measured usage + +Scanned: the generated `opensemantic.*-python` packages plus several +applications built on them (vendored `.venv`, `.tox` and `site-packages` copies +excluded). Application code is referred to generically below; the counts are +what matters for the compatibility decision. + +| member | sites | verdict | +|---|---:|---| +| `json_schema_extra` | 2499 | v2 declaration, already supported unchanged | +| `OswBaseModel` | 246 | subclass of `LinkedBaseModel` | +| `range=` | 187 | **v1 declaration style** (extras land in `field_info.extra`) | +| `get_cls_iri` | 42 | unchanged | +| `get_iri_ref` | 24 | **keep** (see pattern E) | +| `LinkedBaseModel` | 16 | direct base-class references | +| `__iris__` | 15 | **keep, read and write** (pattern C) | +| `from pydantic import` / `.v1 import` | 25 / 14 | **both versions in active use** | +| `to_json` / `from_json` | 9 / 9 | portable | +| `to_jsonld` / `from_jsonld` | 4 / 1 | portable | +| `cast` / `cast_none_to_default` | 3 / 2 | portable | +| `get_raw` | 2 | obsolete (pattern A) | +| `LinkedBaseModelList` | 0 | no downstream use | +| `store_jsonld` | 0 | no downstream use | + +## Why these patterns exist + +Most of them are **workarounds for the shipped binding's hidden I/O**: plain +attribute access may perform a synchronous, un-batchable backend call inside +`__getattribute__`. Callers who cannot afford that, or cannot tell whether a +value is resolved, route around the getter. Under the descriptor binding - +where reads are batched, cached and return real objects - the reason for most +of these disappears. + +### A. The resolved-or-IRI dance - **obsolete** + +Application helpers repeat a shape equivalent to: + +```python +def _load_first_relation(field_name): + raw_value = (entity.get_raw(field_name) + if callable(getattr(entity, "get_raw", None)) + else getattr(entity, field_name, None)) + if raw_value: + return raw_value[0] if isinstance(raw_value, list) else raw_value + relation_ids = (entity.get_iri_ref(field_name) + if callable(getattr(entity, "get_iri_ref", None)) + else None) + ... +``` + +It exists because the caller cannot ask "is this resolved?" without risking +resolution, and must therefore handle both representations. Under the new +binding the whole helper collapses to: + +```python +values = entity.field # real objects, batched and cached +return values[0] if values else None +``` + +Retires `get_raw` (2 sites) entirely. + +### B. `try`/`except` around an attribute read - **obsolete** + +Seen inside a published `opensemantic.*` package: + +```python +try: + char_iri = getattr(channel, "characteristic", None) +except (ValueError, ImportError): + return None +``` + +Catching **`ImportError` from an attribute read** is the clearest symptom of the +problem: the getter resolves, which constructs a type, which may import. With +resolution moved out of the read path this guard has no reason to exist. + +### C. Writing `__iris__` to fabricate a link - **obsolete, but must keep working** + +Also inside a published `opensemantic.*` package: + +```python +self.__iris__ = {"characteristic": characteristic_class.get_cls_iri()} +``` + +A stub object is given a link by writing the internal side-dict, because there +was no clean way to set a link to an IRI. Under the new binding that is simply: + +```python +self.characteristic = characteristic_class.get_cls_iri() # coerced to a link +``` + +Because this is **written**, not just read, a read-only `__iris__` shim would +silently drop the assignment. The compatibility layer therefore implements +`__iris__` as a read/write property. + +### D. Hedging both representations - **simplifies** + +```python +target = self.load_typed(obj.get_iri_ref("some_type") or obj.some_type, SomeType) +``` + +`... or ...` hedges: use the IRI if present, else whatever the attribute holds. +With one predictable representation this becomes a single expression. + +### E. Comparing identity without fetching - **legitimate, keep** + +```python +if module_iri in (subprocess.get_iri_ref("tool") or []): + ... +``` + +Here the caller genuinely wants the IRI, not the object: resolving every related +entity only to compare identity would be wasteful even when batched. This is a +real primitive, not a workaround, so **`get_iri_ref` stays** with its current +name and return shape (`str | list[str] | None`). + +### F. Capability probing - **obsolete** + +```python +entity.get_raw(f) if callable(getattr(entity, "get_raw", None)) else ... +``` + +Defensive checks for whether the API exists at all. A stable base class removes +the need. + +## The metaclass identity must move with the base class + +Found by running a downstream test suite against a swapped binding rather than +by reading code. `opensemantic.characteristics.quantitative._static` does: + +```python +from oold.model import LinkedBaseModelMetaClass as ModelMetaclass # aliased! + +class QuantityValueMetaclass(ModelMetaclass): ... +class QuantityValue(OswBaseModel, metaclass=QuantityValueMetaclass): ... +``` + +Downstream **imports and subclasses oold's metaclass**, aliased to a name that +makes it look like pydantic's. Replacing only `LinkedBaseModel` leaves +`LinkedBaseModelMetaClass` pointing at the old class, so `QuantityValue`'s +metaclass is no longer a subclass of its base's and the import dies with: + +```text +TypeError: metaclass conflict: the metaclass of a derived class must be a +(non-strict) subclass of the metaclasses of all its bases +``` + +The whole package tree fails to import - not a subtle behavioural drift but a +hard failure at collection time. So `LinkedBaseModelMetaClass` is **part of the +public API** and its identity has to be carried over together with the base +class, either by keeping the name bound to the new metaclass or by having the +new metaclass inherit from it. + +Verified: after also rebinding the metaclass, the same suite imports cleanly and +passes. + +## The type registry must be the same object + +The same class of problem, found by enumerating every symbol downstream imports +from `oold`: + +| imported from `oold.model` | sites | +|---|---:| +| `_types` | **7** | +| `LinkedBaseModel` | 2 | +| `LinkedBaseModelMetaClass` (aliased) | 1 | +| `BaseController` | 1 | + +`_types` - the *private* registry - is imported more often than the base class +itself, and it is **written to**: + +```python +from oold.model import _types +_types[SomeClass.get_cls_iri()] = SomeClass +``` + +both in shipped package code and in examples. A replacement that keeps its own +registry dict does not see those entries, so polymorphic resolution silently +falls back to the declared target. Unlike the metaclass conflict this fails +**quietly**, which makes it the more dangerous of the two. + +Fix: share the object, do not copy it - `use_type_registry(oold.model._types)`. + +## How to solve both blockers + +The rule the two findings share: **downstream imports `oold.model` internals by +name and mutates them, so the replacement must preserve names and object +identity, not merely behaviour.** Concretely: + +1. **Name the new metaclass `LinkedBaseModelMetaClass`.** Downstream subclasses + whatever that name resolves to, so pointing it at the new metaclass makes + `class Derived(Base, metaclass=CustomMeta)` consistent by construction. + Inheriting the *old* metaclass from the new one does not work - the derived + metaclass must be a subclass of the base's, not the other way round. +2. **Bind the new registry to the existing `_types` dict** rather than creating + one, so entries written through either name are visible to both. +3. **Keep `LinkedBaseModel` and `BaseController` as the exported names** for the + new implementations. +4. Everything under `oold.backend.*` is untouched by the swap - the remaining + downstream imports (`interface`, `document_store`, `auth`) need no action. + +Both fixes are verified: with the metaclass rebound the previously failing suite +imports and passes, and with the registry shared a downstream registration +resolves through the new binding. + +## Consequences for the replacement + +**Must be preserved** (compatibility layer, `oold/experimental/compat.py`): + +- `get_iri_ref(field)` -> `str | list[str] | None`, unchanged name and shape +- `__iris__`, readable **and assignable** +- `to_json`, `to_jsonld`, `from_json`, `from_jsonld` +- `cast`, `cast_none_to_default`, `get_cls_iri`, `export_schema`, `full_dict` +- both declaration styles: `json_schema_extra={"range": ...}` and bare `range=` +- **`LinkedBaseModelMetaClass`** - downstream subclasses it (see above) +- **`_types`** - the same dict object, downstream writes into it +- **pydantic v1 and v2** + +**May be deprecated once downstream is updated**: `get_raw`, and the defensive +idioms in patterns A, B, C, F. They can be kept as thin shims and removed on a +later major version - none of them needs to survive in the new design on its +own merits. + +**Not needed**: `LinkedBaseModelList` and `store_jsonld` have no downstream +callers, so the rich list operations and the store helper carry no +compatibility obligation (they remain available, but need not constrain the +design). + +## Verification plan + +Parity is asserted only when these pass unchanged against the new base: + +1. the `oold-python` suite, +2. the test suites of the applications that call `get_iri_ref` directly and + assert on its return shape, +3. a regenerated `opensemantic.core` diffed against the released package. + +Status: step 2 has been run with the real switch (not a shim) against three application suites, each compared to a baseline taken +on the same machine and the same backend state: + +| suite | baseline | with the switch | +| --- | --- | --- | +| live-backend controller suite calling `get_iri_ref` | 2 passed | 2 passed | +| dashboard suite | 20 passed | 20 passed | +| utilities suite | 194 passed, 9 failed | 194 passed, 9 failed (same set) | + +The utilities suite fails identically with and without the switch; those +failures predate it. + +On the strength of that, the descriptor binding is now the **default**: +`oold.model.LinkedBaseModel` is the descriptor model, and the legacy binding is +reachable with `OOLD_DESCRIPTOR_BINDING=0`. + +### The shim was not sufficient verification + +Swapping the base class through a `sitecustomize` shim passed; the real switch +failed at import on the first generated package. Four binding defects surfaced +that fixtures had not provoked - a subclass redeclaring an inherited link field, +a link field default that can only raise, `__setattr__(..., internal=True)`, and +`to_json()` leaving `UUID` objects in the v1 path. All are fixed, with +regression tests in `tests/test_downstream_shapes.py`. + +The lesson for the remaining migration steps: verify with the switch, against a +generated package, on a live backend. Two of the four defects were invisible to +every unit test. + +## The metaclass identity is a migration constraint + +Downstream subclasses `LinkedBaseModelMetaClass`, so the name has to keep +resolving to whatever metaclass `LinkedBaseModel` uses. The query DSL built on +that metaclass is carried over unchanged, typed subscription overloads included; +see `graph-object-binding.md`. + +[oold-python#107]: https://github.com/OO-LD/oold-python/issues/107 diff --git a/docs/design/graph-object-binding.md b/docs/design/graph-object-binding.md new file mode 100644 index 0000000..eddeed1 --- /dev/null +++ b/docs/design/graph-object-binding.md @@ -0,0 +1,639 @@ +# Graph-object binding and code generation: a design reflection + +Status: draft for discussion, tracked in [oold-python#107]. + +The recommended binding has been promoted out of the prototypes and now lives in +the package proper: + +| module | what it is | +|---|---| +| `src/oold/model/_descriptor.py` | **the binding.** Descriptors installed from annotations, the `Link[T]` / `LinkList[T]` notation, the query DSL | +| `src/oold/model/v1/_descriptor.py` | the same for pydantic v1 | +| `src/oold/model/_compat.py` | the downstream API surface (`__iris__`, `get_iri_ref`, `to_json` ...) | +| `src/oold/model/_notation.py` | the reviewed notations on top of it: `OoldField()`, union arms | +| `src/oold/experimental/codegen_spike.py` | IR-based code generation without text post-processing (still a spike) | + +It **is** `oold.model.LinkedBaseModel`; the legacy per-attribute-interception +binding is reachable with `OOLD_DESCRIPTOR_BINDING=0`, and +`oold.model.LINK_NOTATIONS_ACTIVE` reports which is in force. + +It took two attempts. The first flip rested on three downstream suites passing, +which was necessary and not sufficient: a review then found seven behaviours the +binding did not reproduce - `BaseController.to_json()` stripping every data +field, `FieldProxy` losing truthiness and default forwarding, +`x-oold-required-iri` enforced nowhere, `__iris__` assignment merging instead of +replacing, `get_raw` answering `[None]`, an inline linked object without an IRI +being dropped, and unset links vanishing from `model_dump()`. None of those +paths were reached by any suite. The default was withdrawn, each was fixed with +a test in `tests/test_compat_parity.py` that fails without its fix, and the +binding was made the default again on that basis rather than on suite results +alone. + +Verification scripts under `examples/`: `check_binding_features.py` (requirement +matrix), `bench_binding_variants.py` (per-operation benchmarks), +`bench_attribute_access.py` (where the interception cost comes from). + +This document asks the questions the OO-LD v0.8 migration hinges on: + +1. Is the current object-graph binding the best approach we can build in Python? +2. Would the problem be easier in another language, and what does that tell us? +3. What would a linked-data-native language look like? +4. Given how much of the toolchain is patch code around + `datamodel-code-generator`, should we write our own generator? + +## 1. What the binding must do + +A property whose value is another entity can be written two ways in the same +field: inline as a nested object, or by reference as an IRI string (annotated +`x-oold-range`, legacy `range`). The binding layer has to **construct** from +either form, **resolve** an IRI lazily through a pluggable backend, +**serialise** references back to IRIs in JSON and JSON-LD, and keep **static +typing** so the declared type is the target model. + +Beyond that minimum, the shipped library also provides polymorphic resolution +(dispatch on the instance type IRI), batched list resolution, rich list +operations, and a class-level query DSL. All of these are requirements, not +extras: they are verified per variant in `check_binding_features.py`. + +## 2. Critique of the current approach + +`src/oold/model/__init__.py` intercepts attribute access unconditionally: + +- **Global monkeypatch.** `pydantic.fields.FieldInfo` is replaced process-wide + at import (`model/__init__.py:57`). Any code importing `oold.model` inherits a + patched pydantic. +- **Metaclass attribute interception.** `LinkedBaseModelMetaClass` overrides + `__getattribute__` (`:157`) for the class-level query DSL, needing a + `_constructing` guard (`:119-132`) to avoid corrupting pydantic's own + metaclass bookkeeping. +- **Instance interception plus a parallel state dict.** Each instance overrides + `__getattribute__` / `__setattr__` (`:625-673`); every read consults the + `__iris__` side-dict (`:399`) and may perform synchronous backend I/O inside + the getter. `__iris__` duplicates field state, forcing heavy `__init__` + special-casing (`:474-582`) and a bespoke list type (`:223`). +- **Import-order-dependent registries** `_types` / `_controller_types` (`:109`). + +### Cost, measured + +`examples/bench_binding_variants.py`, 100k iterations per operation, best of 5, +each variant in its own process. `(x)` is relative to plain pydantic v2 reads. + +| variant | plain read | plain write | link read | link write | query build | +|---|---:|---:|---:|---:|---:| +| plain pydantic v1 | 3.5 (0.7x) | 47.9 (9.8x) | na | na | na | +| plain pydantic v2 | 4.9 (1.0x) | 23.9 (4.9x) | na | na | na | +| gated `__getattribute__` | 19.8 (4.0x) | 69.0 (14.1x) | na | na | na | +| shipped v1 | 58.1 (11.9x) | 959.9 (195.8x) | 426.2 (86.9x) | 1590.4 (324.4x) | 286.9 (58.5x) | +| shipped v2 | 93.3 (19.0x) | 572.5 (116.8x) | 838.1 (171.0x) | 1140.8 (232.7x) | 490.1 (100.0x) | +| **auto-descriptor** | **5.1 (1.0x)** | **50.1 (10.2x)** | **6.3 (1.3x)** | **270.8 (55.2x)** | **194.6 (39.7x)** | +| explicit `Ref[T]` | 4.9 (1.0x) | 24.4 (5.0x) | 5.8 (1.2x) | 27.6 (5.6x) | na | + +Every attribute of every `LinkedBaseModel` - including fields that are not +references - pays roughly 12x (v1) to 19x (v2). + +### Could the interception just be gated on range annotations? + +Partly. Early-exiting for non-link fields removes most of the *work* (19x down +to ~3.1x for an idealised closure-frozenset gate, ~4.5x for a realistic +per-class one), but not the *call*: defining `__getattribute__` at all forces a +Python-level function call on every access instead of the C-level slot. The gate +shrinks the body, not the call. A descriptor is that same gate implemented in C. + +## 3. Python alternatives + +**(a) Per-field descriptors (recommended).** Only `x-oold-range` fields become +descriptors, so plain fields keep native access. Plain attribute access returns +the **real** resolved object, so `isinstance` holds and the value passes +anywhere the target type is expected. + +**(b) Explicit `Ref[T]` (opt-in).** A reference is a first-class value with +`resolve()` / `await aresolve()`. Resolution becomes visible, batchable and +awaitable - none of which the shipped design can express. Cost: `p.knows[0]` is +a `Ref`, not a `Person`, so `isinstance` fails. + +**(c) The trap: do not dress (b) up as (a).** Declaring a `Ref` field as +`Annotated[Person, ...]` makes a checker read it as `Person` while the runtime +value is a `Ref`. `isinstance` is `False`; the static type is not backed by the +runtime value. Rejected. + +### 3.1 The key optimisation: non-data descriptor plus instance-dict cache + +The descriptor is deliberately **non-data** (it defines `__get__` but not +`__set__`) and stores the resolved value in the instance `__dict__`. Because an +instance dict entry shadows a non-data descriptor, every subsequent read is a +plain C-level dict lookup that never re-enters Python - the +`functools.cached_property` pattern. Writes remain intercepted by a targeted +`__setattr__`, which pops the entry to invalidate it. + +Caching in a pydantic `PrivateAttr` instead costs a Python-level `__getattr__` +per read, which is what made link reads slow: + +| link read (warm) | time | vs plain field | +|---|---:|---:| +| data descriptor + `PrivateAttr` cache | 336.0ms | 31.8x | +| **non-data descriptor + `__dict__` cache** | **10.5ms** | **1.00x** | +| plain pydantic field (baseline) | 10.6ms | 1.00x | + +A **32x** improvement on the hot path; link reads drop from 33.6x to 1.3x in the +full matrix. Both descriptor prototypes use it. + +### 3.2 Static typing + +A link has **two** types and one annotation can only state one of them. What you +read is a resolved object; what you may write is that object *or* a reference to +it - an IRI string, or a JSON object still to be constructed. Since pydantic's +`dataclass_transform` takes the annotation as the `__init__` parameter type, +`knows: list[Person]` necessarily rejects `knows=["ex:bob"]`. + +The only mechanism in the typing spec that carries both is the **descriptor +protocol** (PEP 681): when the annotation *is* a descriptor type, a checker takes +the `__init__` parameter and the assignment type from `__set__` and the attribute +type from `__get__`. `Link[T]` and `LinkList[T]` are therefore usable as the +whole annotation: + +```python +class Person(LinkedBaseModel): + knows: LinkList["Person | None"] = OoldField() + employer: Link[Organization] = OoldField() +``` + +`__set__` is declared under `TYPE_CHECKING` only, so at runtime the descriptor +stays **non-data** and the instance-`__dict__` cache from 3.1 is untouched. +`__get_pydantic_core_schema__` builds the schema of the *target*, so the emitted +JSON Schema is byte-identical to the plain annotation - `$ref`, arrays, unions +and forward references included. + +#### Optionality is declared, not assumed + +`Link[T]` reads as `T`, not `T | None`. The declaration is a promise the binding +keeps, so a chain of mandatory links needs no guard per hop: + +```python +class Person(LinkedBaseModel): + father: Link["Person"] = OoldField() # mandatory + mother: Link["Person | None"] = OoldField() # optional + +person.father.father.father.name # no guards - the type says it resolves +mother = person.mother # guard required, and warranted +``` + +Without this, every hop needs an `is not None` check and chaining collapses - +the type would be truthful and useless at once. Handing back a `None` the +declaration denies is the alternative, and that is the same polite fiction the +`Annotated`-over-`Ref` form was rejected for in 3(c). + +What the promise costs, by case: + +| | `Link[T]` | `Link[T \| None]` | +| --- | --- | --- | +| not set, no IRI | **raises `LinkNotResolved`** | `None` | +| backend error | propagates | propagates | +| answered, no such entity | **raises `LinkNotResolved`** | `None` | + +All three fire on *access*, not at construction. An earlier version rejected an +absent mandatory link when the object was built - knowable without resolving +anything, and tempting for that reason - but it over-enforces: the annotation +says what *reading* the link yields, not that every instance carries one. Graph +data is routinely partial (most Wikidata people have no recorded father), and +rejecting those objects makes them unloadable. Declaring a link mandatory states +an intent to **traverse** it, so one `try/except` around a whole walk replaces a +guard at every hop. + +The middle row is unchanged and matters: a transport failure is not "has no +father", and conflating the two would be the real bug. + +#### Coverage + +| spelling | read | write by IRI | runtime | +| --- | --- | --- | --- | +| `Link[Organization]` | `Organization` | typed | mandatory | +| `Link[Organization \| None]` | `Organization \| None` | typed | optional | +| `LinkList["Person"]` | `LinkResultList[Person]` | typed | mandatory elements | +| `LinkList["Person \| None"]` | `LinkResultList[Person \| None]` | typed | optional elements | +| `list["Person"]` | `list[Person]` | not typed | optional | +| `Optional[Organization]` | `Organization \| None` | not typed | optional | +| `knows = LinkList("Person")` | `LinkResultList[Person]` | n/a - no field | optional | + +Every spelling except the two `Link[...]` forms stays optional, so existing +declarations - including everything the generator emits - behave exactly as +before. Consumers that want reference assignment enforced on generated models can +scope the rule per file rather than repo-wide: ty supports +`[[tool.ty.overrides]]` with an `include` glob. + +A to-many link keeps the slot of an unresolvable reference, so the list stays +aligned with the stored references - as `None` when the element type admits it, +otherwise as a raise. A query result is different: it answers with what it found, +so an IRI it could not place is dropped rather than kept. + +Both **pyright and ty** resolve all of it, including `Model[...]` through the +metaclass `__getitem__` overloads. `tests/typing/links.py` and +`tests/typing/query_dsl.py` are checked by both. + +One environment trap is worth knowing, because it fails silently rather than +loudly: if the configured environment cannot resolve pydantic, ty reports a +spurious `conflicting-metaclass` on every model and then infers `Unknown` for +class subscription - so every `assert_type` passes vacuously. `tests/test_typing.py` +therefore points ty at the interpreter running the tests, not at `./.venv`. + +### 3.3 Declaration notations + +**Recommended: `Link[T]` / `LinkList[T]` as the whole annotation, with a bare +`OoldField()`.** + +```python +class Person(LinkedBaseModel): + id: str + employer: Link["Organization | None"] = OoldField() + knows: LinkList["Person"] = OoldField() +``` + +The annotation is the single source of truth: it names the target, says the +field is a link, and declares optionality. So `range=` is not passed - it would +state the target twice and let the two disagree - and `link=True` is redundant. +`x-oold-range` is derived from the annotation when the schema is generated, +which is what keeps the emitted schema a link schema. + +Deriving rather than repeating had to wait for the right moment to do it. A +forward reference is not resolvable when the descriptor is installed, and a +`Field()` object is shared between models, so its `json_schema_extra` must not +be mutated in place. `__get_pydantic_json_schema__` has neither problem. + +Every other notation is supported and keeps working - including the legacy +`Field(None, json_schema_extra={"range": ...})` that code generation still +emits - but each gives something up, and the table after the example says what. +`Link[T]` / `LinkList[T]` are pydantic v2 only; `oold.model.v1` keeps the +`range=` form. + +Four notations are supported; all share one descriptor implementation, and they +can be mixed in a single class. + +```python +class Person(OoldModel): + id: str + name: Optional[str] = None + + # 1. Link[T] / LinkList[T] as the whole annotation - typed both ways (3.2) + knows: LinkList["Person"] = OoldField() + employer: Link[Organization] = OoldField() + + # 2. implicit, zero-config - target inferred from the annotation + friends: Optional[List["Person"]] = OoldField() + + # 3. union arms: literal text | inline object | reference + location: Union[str, Location, None] = OoldField(link=True) + + # 4. unannotated descriptor - no annotation, so no static type at all + # addresses = LinkList(Address) +``` + +All four are the same field at runtime and produce the same JSON Schema; they +differ only in what a type checker can see, per the coverage table in 3.2. The +union arms discriminate at +construction: a bare string stays a literal when a `str` arm is declared, a +`{"@id": ...}` object is a reference, and any other object is inline. An inline +object with no `@id` cannot be emitted as a reference, so it serialises nested - +a blank node. + +#### Which notation supports what + +Every row is the same field at runtime - they resolve, batch, serialise and +query identically. They differ in what a type checker sees and what reaches the +JSON Schema. Measured, not asserted: read types from `ty`, schema keys from +`model_json_schema()`. + +| notation | codegen emits it | target inferred | range keyword in schema | read type | IRI write typed | optionality declarable | +|---|---|---|---|---|---|---| +| `Optional[List[T]] = Field(None, json_schema_extra={"range": ...})` | yes | no | `range` (legacy) | `list[T] \| None` | no | no | +| `= OoldField(range="...")` | no | no | `x-oold-range` as given | as annotated | no | no | +| `= OoldField()` | no | **yes** | **`x-oold-range`, derived** | as annotated | no | no | +| `Link[T]` / `LinkList[T]` with `OoldField()` | not yet | **yes** | **`x-oold-range`, derived** | **exact** (`T`, `LinkResultList[T]`) | **yes** | **yes** | +| `= Link(T)` / `= LinkList(T)` | no | yes (from the argument) | **field absent from schema** | exact | n/a - not a field | no | +| `str \| Location \| None = OoldField(link=True)` | no | yes | **derived** | union as declared | no | via the `None` arm | + +The derived range is the target's `get_cls_iri()`, taken at schema-generation +time; an explicit `range=` is never overwritten, and a target that cannot name +itself contributes nothing, leaving the `x-oold-link` marker in place. The +unannotated descriptor form is not a pydantic field at all, so it neither +appears in the schema nor gets an `__init__` parameter, though its read type is +exact. + +#### Requiredness is a field argument, not the annotation + +Two different questions - "must the caller supply it?" and "what do I get when I +read it?" - so two carriers: + +| declaration | stored as | enforced | on violation | +|---|---|---|---| +| `Link[T]` - no `None` arm | `_AutoLink.optional = False` | on read | `LinkNotResolved` | +| `OoldField(required=True)` | `_AutoLink.required_iri`, precomputed into `cls.__required_links__`; emitted as `x-oold-required-iri` and into the standard `required` array | in `__init__` | `ValueError: ... is required but not set` | + +A link is never required at the *pydantic* level, because its value is routed +out of the payload before validation - which is why the legacy binding declared +every generated link field `Optional[...]` and carried requiredness in the +keyword. A bare `Link[T]` annotation with no default is read as required, which +is what Python means by "no default" everywhere else. + +**Why not put requiredness in the annotation.** `required` -> `Link[T]`, +absence -> `Link[T | None]` reads well and was the first proposal. It fails on a +self-referential link. `father: Link["Person"]` required means every person in +the dataset carries a father IRI, which is only true of a graph with no root - +and resolution constructs target objects, so the failure surfaces one hop from +its cause: reading `alice.father` raises `ValueError: father is required but not +set` about *Bob's* document, which you never asked for. So `father` would have +to be `Link["Person | None"]`, which needs a guard per hop - the thing the +annotation exists to avoid. + +The alternative was to strip the `| None` under `TYPE_CHECKING`, which both +pyright and ty do resolve (a self-type overload on `Link[X | None]` yields `X`). +It was rejected because the runtime would then have to raise instead of +returning `None`, which silently breaks `entity.link is None`, `if entity.link:` +and `getattr(entity, f, None)` - and the last does *not* save the caller, since +`LinkNotResolved` is a `LookupError`, not an `AttributeError`. Pattern A in +`downstream-migration.md` is exactly that shape. + +Suppressing the diagnostic instead is only half-available: pyright separates +`reportOptionalMemberAccess` from `reportAttributeAccessIssue`, so it can be +turned off without losing typo detection, but ty reports both under +`unresolved-attribute` (checked on 0.0.49 and 0.0.80). Either way it is a +setting every downstream consumer would have to make. + +**Open:** nothing emits the read-side promise, so a model -> schema -> model +round trip loses the distinction between `Link[T]` and `Link[T | None]`. Code +generation can default to `Link[T]` for required properties and +`Link[T | None]` otherwise; expressing it exactly needs a keyword of its own. + +#### Notations considered and dropped + +| notation | why it is not used | +|---|---| +| `list[Link[T]]` - `Link` nested inside a container | Silently degrades. A descriptor nested in a `list` is not treated as one, so the read type comes back as `list[Link[T]]` - not merely untyped but **wrong**. Superseded by `LinkList[T]`, which carries the to-many-ness itself. Still works at runtime, which is what makes it dangerous. | +| `Optional[Link[T]]` - `Link` nested inside a union | Half-degrades: the read type narrows to `T \| None` correctly, but the descriptor's `__set__` is not seen through the wrapper, so assigning an IRI is rejected (`Expected Link[T] \| None`). `Link[T \| None]` states the same thing and types both directions. | +| `Annotated[Person, OoldRange(...)]` wrapping a `Ref` value | The static type is not backed by the runtime value: a checker reads `Person`, `isinstance` says `Ref`. Rejected in 3(c). | +| `Ref[T]` as the field type | Honest, but `p.knows[0]` is a `Ref`, not a `Person` - `isinstance` fails and the list operations, polymorphic dispatch and query DSL go with it (3.6). Kept only as an opt-in handle for visible or async resolution. | +| `~Person.name == "x"` for match filters | `~` binds tighter than `==`, so this parses and would work - but pandas established `~` as NOT, and colliding with that is worse than a method. | + +Two **semantics** were dropped along the way, for the record: elements of a +to-many link were briefly `T | None` unconditionally (replaced by declaring it), +and a mandatory link was briefly rejected at construction rather than on access +(see the table above). + +### 3.4 Typed `json_schema_extra` + +The raw dict can be replaced by a validated class, but it **must subclass +`dict`**: pydantic merges extras via `isinstance(json_schema_extra, dict)`, so a +plain `BaseModel` is accepted at declaration and then silently dropped from the +schema. `OoldExtra` delegates validation to a pydantic model and exposes typed +properties, so runtime code stops doing `extra["x-oold-range"]`. + +```python +OoldExtra(range="") # ValidationError: String should have at least 1 character +extra.range # typed read (str) +``` + +Pass the payload to `model_validate` as a dict rather than as aliased kwargs, +otherwise type checkers reject `range=` as "No parameter named". + +### 3.5 Query DSL + +Preserved, and cheaper. It moves from `__getattribute__` (every access) to +`__getattr__` (a fallback, only when lookup *fails*). Pydantic v2 removes field +names from the class namespace, so `Person.name` fails naturally and lands +there at no cost to anything else. For link fields no metaclass is involved at +all: the descriptor's `__get__(None, owner)` returns the descriptor on class +access, so comparison operators live directly on it. + +`__getattr__` on the metaclass must never call `getattr(cls, ...)`: +`cls.model_fields` is a property that itself calls `getattr`, which recurses +until the stack overflows. Read `klass.__dict__["__pydantic_fields__"]` along +the MRO and reject `_`-prefixed names. + +**Static types.** The condition expression itself cannot be typed: `Entity.name` +is a declared field, so pydantic's `dataclass_transform` makes the annotation +authoritative for class-level access too and the `FieldProxy` really returned is +invisible, leaving `Entity.name == "x"` as `bool`. The shipped binding handles +this by accepting `bool` in the `__getitem__` overloads, and the descriptor +binding does the same: the argument type stays wrong, the result type comes out +right. What the subscript yields is fully typed: + +| expression | static type | +| --- | --- | +| `Entity["ex:e1"]` | `Entity \| None` | +| `Entity[Entity.name == "x"]` | `LinkResultList[Entity] \| None` | +| `...[0]` | `Entity` | +| `...[cond]` | `LinkResultList[Entity]` | + +`LinkResultList` is generic in the item type, which is what keeps the item type +through indexing and filtering. Accepting `bool` there widens +`list.__getitem__`, which answers `T` for a `bool` index - a deliberate +divergence, since indexing a list by `True` is not something anyone writes. +`tests/typing/query_dsl.py` pins these with `assert_type`; +`tests/test_typing.py` runs pyright over it. + +The `| None` is the one difference from the shipped overloads, which promise a +bare `M`: the query really does return `None` when nothing matches, so the +stricter type is the truthful one. + +Instance-level filtering (`entity.links[cond]`) is typed only when the field is +annotated `LinkResultList[T]`. The `list[T] | None` form the current codegen +emits stays unfiltered at the type level, so the generator should emit +`LinkResultList[T]` for to-many links. + +#### The DSL against a real query language + +A `Condition` is consumed by two things, and the check that matters is that they +agree: `apply_operator`, which filters objects already in memory, and the SPARQL +resolvers, which have to ask a triple store the same question. `query()` on +`SparqlResolver`, `LocalSparqlResolver` and `WikiDataSparqlResolver` translates +`eq, ne, lt, le, gt, ge` and `&`; anything else raises rather than quietly +returning the wrong rows. + +Two things fell out of doing it, which is why it was worth doing before adding +more operators: + +- **The predicate and the literal both come from the model's own context.** A + probe document is expanded through it, so a term scoped `@language: en` yields + `"x"@en` and one with an `@type` yields `"x"^^`. Deriving either + by hand would be a second, divergent reading of the same context. +- **A match needs a type constraint.** `rdfs:label "Tim Berners-Lee"@en` also + matches a book edition, and resolving that fails on an unknown type IRI. The + Wikidata resolver constrains on P31 - the predicate it already rewrites into + `@type` on the way in. + +`tests/test_sparql_query.py` asserts the SPARQL answer against `apply_operator` +over the same data rather than against hand-written expectations, so it tests +the agreement rather than one implementation twice. It runs offline against an +rdflib graph. + +Still missing, and visible from here: there is no `|` (the model defines +`__and__` but not `__or__`), no `~`, and link descriptors carry only `==` / `!=`, +not the ordering operators. + +### 3.6 Requirement matrix + +From `examples/check_binding_features.py`, which exercises each requirement +rather than asserting it. + +| requirement | shipped v1 | shipped v2 | auto (implicit) | auto (explicit) | `Ref[T]` | +|---|---|---|---|---|---| +| syntax_unchanged | ok | ok | ok | FAIL | FAIL | +| build_by_iri / build_by_object | ok | ok | ok | ok | ok | +| lazy | ok | ok | ok | ok | ok | +| real_object (`isinstance`) | ok | ok | ok | ok | FAIL | +| polymorphic | ok | ok | ok | ok | FAIL | +| batched | ok | ok | ok | ok | FAIL | +| cached | ok | ok | ok | ok | ok | +| mutation | ok | ok | ok | ok | FAIL | +| link_validated | ok | ok | ok | ok | ok | +| list_lookup / list_filter | ok | ok | ok | ok | FAIL | +| list_projection | FAIL | FAIL | **ok** | ok | FAIL | +| serialize_iri | ok | ok | ok | ok | ok | +| query_dsl | ok | ok | ok | ok | FAIL | +| typed_extras | FAIL | FAIL | **ok** | ok | FAIL | +| no_monkeypatch | FAIL | FAIL | **ok** | ok | ok | + +The descriptor binding is a **strict superset** of the shipped one. Note the +shipped implementation *does* batch list resolution - an earlier claim to the +contrary was wrong. + +On validation: the parent's *field* validation is bypassed (the descriptor +shadows the field), but the linked object is still validated **at construction +of the linked class**, which is where its constraints live. + +## 4. Would another language do better? + +The shipped design makes *every* attribute transparently resolve. Python has no +cheap whole-object proxy, so that choice forces `__getattribute__` plus a +metaclass. Per-field descriptors avoid it entirely. + +- **JavaScript / TypeScript.** `Proxy` is a language primitive, so transparent + lazy references are idiomatic and cheap (LDO, rdf-ts). +- **Rust / TreeLDR.** Compiles a linked-data schema to typed Rust plus a JSON-LD + context; references are an id newtype (`IdRef`) and resolution is explicit + I/O - essentially the `Ref[T]` design enforced by the type system. +- **Java / twa, TheWorldAvatar OGM.** Annotation-driven mapping resolving + through a session object; explicit, not getter-side-effect. +- **Clojure / Datomic.** No object graph at all: entity-attribute-value tuples, + a reference is an entity id, and resolution is an explicit `pull` with a + declared shape. + +Every ecosystem that handles this well makes resolution **explicit** or has a +**language-level proxy**. Python has neither at whole-object level, but the +descriptor protocol is exactly the per-field equivalent, and `Ref[T]` covers the +explicit camp. Supporting both matches the two durable designs rather than +picking one. + +## 5. What would a linked-data-native language look like? + +- **IRIs and language-tagged strings as primitive types**, not `str`. +- **Identity and type first-class on every value**; open-world structural typing + aligned to SHACL shapes rather than closed classes. +- **Lexical namespaces / contexts**, so `name` resolving to `schema:name` is a + compile-time fact. +- **References transparent, resolution an effect.** Reading a linked value is + ordinary syntax, but resolution is tracked by an effect / capability (like + `async`) served by a pluggable resolver: transparency without hidden, untyped + I/O. +- **Graph literals and query comprehensions** as language constructs. +- **Built-in JSON-LD / RDF serialisation**, because the object model *is* the + RDF model. + +Prior art: N3, Shen, LinkML, TreeLDR, RDF-star, Datomic/Datalog, GraphQL-LD. + +The recommended binding already approximates the ideal: reads look ordinary and +return real objects, while `.refs()` / `aresolve()` expose resolution as an +explicit, batchable, awaitable effect. What Python cannot have - IRIs as +primitives, structural open-world typing - is exactly what argues for keeping +the source of truth in the schema and generating the binding from it. + +## 6. Own generator vs datamodel-code-generator + +Code generation currently fights the tool from both ends: `src/oold/generator.py` +monkeypatches the parser and regex-fixes its output; `src/oold/utils/codegen.py` +subclasses it to inject `@context` and repair `allOf`; and the external +`osw-python-package-generator` post-processes the generated *text* with roughly +1100 lines of regex (`_fix_missing_allof_bases`, +`replace_duplicated_classes_with_imports`, UUID/OSW-ID dedup, +`replace_unit_enums`). + +Those structural problems - multiple `allOf` inheritance, class identity by +`x-oold-uuid`, cross-package imports, typed `x-oold-range` references - are +graph facts the tool does not model, so they are fought after the fact on +strings. + +Options: (1) keep and patch, (2) hybrid - keep the tool for plain JSON Schema +fragments but operate on its *model objects* instead of text, (3) own IR-based +generator. + +`codegen_spike.py` implements a minimal option 3 and shows the three +regex-fought problems fall out for free from an IR: `allOf` becomes real +multiple inheritance, two schemas sharing `x-oold-uuid` collapse to one class, +and `x-oold-range` becomes a typed reference field - with no post-processing. + +**Recommendation: option 3, staged through option 2.** The real cost is +re-implementing the JSON Schema breadth the tool gives for free (unions, enums, +constraints, formats, naming). Mitigations: stage through the hybrid; gate the +switch on a golden-file diff of a regenerated real package; keep the IR +language-agnostic so it can later emit TypeScript or Rust. + +## 7. CPython and Rust optimisation potential + +Applied: the non-data descriptor plus instance-dict cache (section 3.1), a 32x +win that puts warm link reads at native speed. + +Remaining CPython headroom: + +- **Query construction, ~6.6x available.** `Condition` is a pydantic + `BaseModel`, so every `Cls.field == value` pays full validation: 200.1ms vs + 30.4ms for a `__slots__` class over 200k iterations. It touches the public + `oold.backend.interface` API, so it is a deliberate change. +- **Link writes (55.2x)**, dominated by `Ref` construction and private-attr + access on the write path. +- **Plain writes (10.2x vs 4.9x)**, entirely the extra `__setattr__` frame; + classes with no link fields need no override at all. + +Rust: after the fix above the binding hot path is a C-level dict lookup and +pydantic's validation core is already Rust, so there is little left to win +there. The real opportunities are elsewhere: **`pyld` is pure Python and +dominates RDF export** (`to_jsonld()` 120.7 us/op vs `to_json()` 19.7 us/op, a +6x gap, essentially all context expansion), and `rdflib` is likewise pure Python +where `pyoxigraph` (Rust) is an alternative for graph storage and SPARQL. +Priority: the JSON-LD/RDF layer, not the object binding. + +## 8. Recommendations and impact on the migration + +- **Binding:** adopt the per-field descriptor design, declared by annotation + (unchanged syntax, so generated packages are untouched), with the explicit + descriptor and `Link[T]` notations available and `Ref[T]` as an opt-in handle + for visible or async resolution. Avoid the `Annotated`-over-`Ref` form. +- **Query DSL:** carried over whole, including the typed subscription overloads + (3.5). The `_constructing` guard the metaclass carries is not a cost of the + DSL - the descriptor needs it independently, since a descriptor is an ordinary + class attribute that pydantic's base-attribute probe trips over without + `__getattr__` ever being consulted. +- **Code generation:** move off text post-processing toward an IR-based + generator, staging through a hybrid that first deletes the regex. Fold + `osw-python`'s `fetch_schema` orchestration and the package-generator passes + into it. Emit `LinkResultList[T]` for to-many links so instance-level + filtering type-checks. +- **Sequencing into v0.8:** the keyword migration should read `x-oold-range` / + `x-oold-iri` / `x-oold-uuid` (dual-read with legacy) through the same IR and + binding, so the generator, the runtime binding and the RDF layer share one + keyword-normalisation path rather than three. + +### Open items + +- **pydantic v1**: the prototypes are v2-only + (`__pydantic_init_subclass__`, core schemas); `model/v1/__init__.py` is a full + parallel implementation and the package generator emits both. +- **Public-API equivalence** with the shipped `LinkedBaseModel` (`to_json`, + `to_jsonld`, `from_json`, `from_jsonld`, `cast`, controllers, `Model["iri"]`) + must be demonstrated before adoption so `osw-python` is unaffected. +- **The class registry is still a process-wide global** keyed by type IRI, so + two classes claiming the same IRI shadow each other and resolution depends on + import order. The prototype reproduces the very flaw criticised in section 2; + it surfaced as a cross-module test collision and is currently worked around by + using distinct IRIs per test module. A scoped registry (per root model or per + explicit registry object, with the global as a default) is needed before + adoption. + +[oold-python#107]: https://github.com/OO-LD/oold-python/issues/107 diff --git a/docs/how-to/backends.md b/docs/how-to/backends.md index bbac909..5453a94 100644 --- a/docs/how-to/backends.md +++ b/docs/how-to/backends.md @@ -120,6 +120,33 @@ The backend serializes each entity to JSON-LD before inserting it into the RDF g --- +## Querying a SPARQL backend + +`SparqlResolver`, `LocalSparqlResolver` and `WikiDataSparqlResolver` translate +the query DSL into SPARQL, so `Model[Model.field == value]` reaches the store: + +```python +from oold.backend.sparql import WikiDataSparqlResolver +from oold.backend.interface import SetResolverParam, set_resolver + +set_resolver(SetResolverParam(iri="Item", resolver=WikiDataSparqlResolver())) +Person[Person.name == "Tim Berners-Lee"] +``` + +`eq`, `ne`, `lt`, `le`, `gt`, `ge` and `&` are supported. Anything else raises +rather than returning the wrong rows. The predicate and the literal are taken +from the model's own JSON-LD context, so a term scoped to a language produces a +language-tagged literal and one with an `@type` produces a typed one. + +`WikiDataSparqlResolver` additionally constrains matches to the model's class - +a label matches more than one kind of thing. + +!!! note "Public endpoints want a user agent" + Wikidata answers the SPARQLWrapper default with + `429 Aggressively rate-limiting to 1 req / min`, which looks like an outage + rather than a policy. The resolvers send a descriptive `user_agent` by + default; override it with your own tool name and contact URL. + ## Multiple backends Register different backends for different IRI prefixes: diff --git a/docs/how-to/object-graph-mapping.md b/docs/how-to/object-graph-mapping.md index 89bce2c..40ae918 100644 --- a/docs/how-to/object-graph-mapping.md +++ b/docs/how-to/object-graph-mapping.md @@ -1,6 +1,49 @@ # Object Graph Mapping -oold-python's core feature is *IRI-transparent references*: a field annotated with `range` can hold either a Python object or an IRI string. The library resolves IRIs on first access via the registered backend. +oold-python's core feature is *IRI-transparent references*: a link field can hold either a Python object or an IRI string. The library resolves IRIs on first access via the registered backend. + +## Recommended declaration + +Declare a link with `Link[T]` or `LinkList[T]`, and `OoldField()` with no +arguments: + +```python +from oold.model import Link, LinkedBaseModel, LinkList, OoldField + +class Person(LinkedBaseModel): + id: str + name: str | None = None + employer: Link["Organization | None"] = OoldField() + knows: LinkList["Person"] = OoldField() +``` + +The annotation is the single source of truth. It names the target, so +`x-oold-range` is derived from it and written into the emitted schema - passing +`range=` would state the same thing twice and let the two disagree. It says the +field is a link, so `link=True` is redundant. And it declares optionality: +`Link[T]` reads as `T`, `Link[T | None]` as `T | None`. + +This is also the only form a type checker reads correctly in **both** +directions - the resolved object you get back, and the object, IRI or JSON +object you may assign. + +Where the annotation cannot say it - a union arm such as +`str | Location | None` - mark the field with `OoldField(link=True)`. + +Still supported, not recommended for new code: + +| form | why not | +|---|---| +| `Optional[Bar] = Field(None, json_schema_extra={"range": "Bar.json"})` | the legacy notation, and what code generation still emits. Untyped in both directions | +| `Optional[Bar] = OoldField(range="Bar.json")` | repeats what the annotation already says | + +Nothing existing needs rewriting; the recommendation applies to code you write +now. `Link[T]` / `LinkList[T]` require pydantic v2 - under `oold.model.v1` use +the `range=` form. + +[Typed link declarations](#typed-link-declarations) explains the typing; the +full comparison is in +[the design doc](../design/graph-object-binding.md#which-notation-supports-what). --- @@ -155,3 +198,122 @@ A field can carry a default IRI that is resolved automatically on instantiation: ``` When you instantiate the model without supplying `b_default`, the IRI `"ex:tag-python"` is used and resolved on first access. + +--- + +## Typed link declarations + +The declaration above works, but a type checker only sees half of it. A link has +**two** types: reading it yields a resolved object, while writing it accepts that +object *or* a reference to it - an IRI string, or a JSON object still to be +constructed. A single annotation can only state one, so `knows: list[Person]` +rejects `knows=["ex:bob"]` even though the library accepts it at runtime. + +`Link[T]` and `LinkList[T]` carry both. They are exported from `oold.model`: + +```python +from oold.model import Link, LinkedBaseModel, LinkList, OoldField + +class Person(LinkedBaseModel): + id: str + name: str | None = None + employer: Link["Organization | None"] = OoldField() + knows: LinkList["Person"] = OoldField() + +# accepted: an object, an IRI, or a JSON object +alice = Person(id="ex:alice", knows=["ex:bob", {"id": "ex:carol"}]) +alice.knows[0] # a Person, not a str +``` + +Nothing changes at runtime - same resolution, same JSON Schema. Only what the +checker sees changes. + +### Optionality is declared + +`Link[T]` reads as `T`, so a chain needs no guard at every hop. `Link[T | None]` +reads as `T | None`, because absence is then part of the model: + +```python +class Person(LinkedBaseModel): + father: Link["Person"] = OoldField() # promises a Person + mother: Link["Person | None"] = OoldField() # may legitimately be absent + +person.father.father.father.name # no guards +``` + +A link declared mandatory raises `LinkNotResolved` when it is unset or when the +backend cannot place the reference, so one `try/except` covers a whole walk: + +```python +from oold.model import LinkNotResolved + +try: + while True: + person = person.father + print(person.name) +except LinkNotResolved: + print("ancestry ends here") +``` + +A **transport failure is not absence** - a connection error propagates unchanged +rather than being reported as a missing link. + +### `OoldField` arguments + +All keyword-only, all optional: + +```python +OoldField(required=None, range=None, link=None, **field_kwargs) +``` + +| argument | effect | +|---|---| +| `required` | the link must be supplied at construction; omitting it raises `ValueError`. Emitted as `x-oold-required-iri` **and** into the standard `required` array | +| `range` | target schema IRI, emitted as `x-oold-range`. **Do not pass it**: omitted, it is derived from the annotation, which already names the target | +| `link` | marks the field a link where the annotation does not imply it, as in a union arm. Redundant with `Link[T]` / `LinkList[T]` | +| `required_iri` | deprecated spelling of `required`, kept because generated packages pass it. Same emitted keyword | +| `**field_kwargs` | passed to `pydantic.Field` (`alias`, `description`, `default_factory`, ...). `default=None` is supplied unless you pass a `default_factory` | + +A link annotation with **no default at all** means required, as it does anywhere +else in Python: + +```python +manager: Link[Organization] # required +manager: Link[Organization] = OoldField() # optional +manager: Link[Organization] = OoldField(required=True) # required, explicit +``` + +### Requiredness is a field argument, not the annotation + +"Must the caller supply it?" and "what do I get when I read it?" are different +questions, and they need separate carriers - a self-referential link needs them +to differ. All four combinations are available: + +| | `OoldField()` | `OoldField(required=True)` | +|---|---|---| +| `Link[T]` | may omit; reading raises `LinkNotResolved` if absent | must supply; reading raises if absent | +| `Link[T \| None]` | may omit; reading yields `None` | must supply; reading may still yield `None` | + +```python +class Person(LinkedBaseModel): + father: Link["Person"] = OoldField() # chain it, no guards + employer: Link["Organization"] = OoldField(required=True) + advisor: Link["Person | None"] = OoldField() # guard it +``` + +`father` is the case that forces the split. Reading it must yield a `Person` so +that `person.father.father.father` needs no guard per hop - but no real dataset +can require *every* person to name a father, so it cannot be required at +construction. Requiredness in the annotation would tie those together. + +A link is never required at the *pydantic* level, because its value is routed +out of the payload before validation. That is what `required` exists to express, +and why it is also written into the schema's `required` array - otherwise the +constraint would be invisible to any plain JSON Schema validator. + +!!! note "Write the whole annotation" + Spell the union inside: `Link[T | None]`, not `Optional[Link[T]]`, and + `LinkList[T]`, not `list[Link[T]]`. Nested in another annotation a checker + stops applying descriptor rules - `list[Link[T]]` reads as a list of + descriptors, and `Optional[Link[T]]` narrows on read but rejects an IRI on + write. Both keep working at runtime, which is what makes them easy to miss. diff --git a/examples/backend_auth.py b/examples/backend_auth.py index 0b7a94b..ab9c152 100644 --- a/examples/backend_auth.py +++ b/examples/backend_auth.py @@ -1,14 +1,14 @@ -from oold.backend.auth import SetCredentialParam, UserPwdCredential, set_credential +from pydantic import SecretStr + +from oold.backend.auth import UserPwdCredential, set_credential from oold.backend.sparql import SparqlResolver # ToDo: allow other ways, e.g. environment variables, keyring, ... set_credential( - SetCredentialParam( - credential=UserPwdCredential( - iri="https://blazegraph.kiprobatt.de", - username="user", - password="*********", - ) + UserPwdCredential( + iri="https://blazegraph.kiprobatt.de", + username="user", + password=SecretStr("*********"), ) ) diff --git a/examples/bench_attribute_access.py b/examples/bench_attribute_access.py new file mode 100644 index 0000000..899276a --- /dev/null +++ b/examples/bench_attribute_access.py @@ -0,0 +1,165 @@ +"""Where does the attribute-access cost of the shipped binding come from? + +Decomposes the per-access overhead of ``oold.model.LinkedBaseModel`` into the +cost of *calling* a Python-level ``__getattribute__`` at all, versus the cost of +the *work* it does. This answers whether gating the interception on range +annotations (early-exit for non-link fields) would restore native performance. + +Baselines cover plain pydantic v1 and v2, and both shipped ``LinkedBaseModel`` +variants, so each binding is compared against its own pydantic version. + +Run it: + + python examples/bench_attribute_access.py + +Each variant runs in a **separate subprocess**: importing ``oold.model`` +monkeypatches ``pydantic.fields.FieldInfo`` process-wide, which would otherwise +contaminate the plain-pydantic baselines measured in the same process. + +Result (see docs/design/graph-object-binding.md section 2): gating removes most +but not all of the overhead, because a Python-level ``__getattribute__`` is +still invoked on every access. The descriptor binding reaches parity with plain +pydantic because the equivalent gate is performed by the C-level descriptor +protocol during normal attribute lookup. +""" + +import subprocess +import sys +import timeit + +N = 300_000 +REP = 7 + + +def build_plain_v2(): + from pydantic import BaseModel + + class M(BaseModel): + id: str + literal: str | None = None + + return M(id="x", literal="v") + + +def build_plain_v1(): + from pydantic.v1 import BaseModel + + class M(BaseModel): + id: str + literal: str | None = None + + return M(id="x", literal="v") + + +def build_gated_best(): + """Best case gate: closure frozenset, no attribute lookup on the fast path.""" + from pydantic import BaseModel + + links = frozenset({"link_a", "link_b"}) + + class M(BaseModel): + id: str + literal: str | None = None + + def __getattribute__(self, name): + if name in links: + pass # slow path, not taken for plain fields + return object.__getattribute__(self, name) + + return M(id="x", literal="v") + + +def build_gated_real(): + """Realistic gate: per-class set, needs a type(self) lookup on every access.""" + from pydantic import BaseModel + + class M(BaseModel): + id: str + literal: str | None = None + __link_names__ = frozenset({"link_a", "link_b"}) + + def __getattribute__(self, name): + if name in type(self).__link_names__: + pass + return object.__getattribute__(self, name) + + return M(id="x", literal="v") + + +def build_shipped_v2(): + from oold.model import _LinkedBaseModelLegacy as LinkedBaseModel + + class M(LinkedBaseModel): + id: str + literal: str | None = None + + return M(id="x", literal="v") + + +def build_shipped_v1(): + from oold.model.v1 import _LinkedBaseModelLegacy as LinkedBaseModel + + class M(LinkedBaseModel): + id: str + literal: str | None = None + + return M(id="x", literal="v") + + +def build_auto_descriptor(): + """Auto-installed descriptors: unchanged declaration syntax.""" + + from pydantic import Field + + from oold.model._descriptor import LinkedBaseModel + + class M(LinkedBaseModel): + id: str + literal: str | None = None + links: list["M"] | None = Field(None, json_schema_extra={"x-oold-range": "M"}) + + M.model_rebuild() + return M(id="x", literal="v") + + +VARIANTS = { + "plain_v1": ("plain pydantic v1 (baseline v1)", build_plain_v1), + "plain_v2": ("plain pydantic v2 (baseline v2)", build_plain_v2), + "gated_best": ("gated __getattribute__ (best case)", build_gated_best), + "gated_real": ("gated __getattribute__ (realistic)", build_gated_real), + "shipped_v1": ("shipped LinkedBaseModel v1", build_shipped_v1), + "shipped_v2": ("shipped LinkedBaseModel v2", build_shipped_v2), + "auto_descriptor": ("auto-descriptor, syntax unchanged", build_auto_descriptor), +} + + +def measure(key: str) -> float: + obj = VARIANTS[key][1]() + return min(timeit.repeat(lambda: obj.literal, number=N, repeat=REP)) + + +def main() -> None: + results = {} + for key in VARIANTS: + out = subprocess.run([sys.executable, __file__, key], capture_output=True, text=True) + if out.returncode != 0: + print(f"{key}: FAILED\n{out.stderr[-600:]}") + continue + results[key] = float(out.stdout.strip()) + + b1 = results.get("plain_v1") + b2 = results.get("plain_v2") + print(f"plain-field access, {N:,}x, best of {REP}, each in its own process\n") + print(f"{'variant':40} {'time':>9} {'vs v1':>8} {'vs v2':>8}") + for key, t in results.items(): + label = VARIANTS[key][0] + r1 = f"{t / b1:6.2f}x" if b1 else " na" + r2 = f"{t / b2:6.2f}x" if b2 else " na" + print(f"{label:40} {t * 1e3:7.1f}ms {r1:>8} {r2:>8}") + + +if __name__ == "__main__": + if len(sys.argv) > 1: + print(measure(sys.argv[1])) + else: + main() diff --git a/examples/bench_binding_variants.py b/examples/bench_binding_variants.py new file mode 100644 index 0000000..d32f8d2 --- /dev/null +++ b/examples/bench_binding_variants.py @@ -0,0 +1,221 @@ +"""Benchmark every graph-object binding variant across every hot operation. + +Operations measured: plain-field read, plain-field write, linked-field read +(warm, already resolved), linked-field write, and query construction +(``Cls.field == value``). + +Each variant runs in a **separate subprocess**: importing ``oold.model`` +monkeypatches ``pydantic.fields.FieldInfo`` process-wide and would otherwise +contaminate the plain-pydantic baselines measured in the same process. + +Run it: + + python examples/bench_binding_variants.py +""" + +import subprocess +import sys +import timeit + +N = 100_000 +REP = 5 + + +def build_plain_v1(): + from pydantic.v1 import BaseModel + + class M(BaseModel): + id: str + literal: str | None = None + + return M(id="x", literal="v"), M, None, None + + +def build_plain_v2(): + from pydantic import BaseModel + + class M(BaseModel): + id: str + literal: str | None = None + + return M(id="x", literal="v"), M, None, None + + +def build_gated(): + """Hypothetical: current design with the interception gated on link names.""" + from pydantic import BaseModel + + links = frozenset({"link"}) + + class M(BaseModel): + id: str + literal: str | None = None + + def __getattribute__(self, name): + if name in links: + pass # slow path, not taken for plain fields + return object.__getattribute__(self, name) + + return M(id="x", literal="v"), M, None, None + + +def build_shipped_v1(): + from pydantic.v1 import Field as F1 + + from oold.model.v1 import _LinkedBaseModelLegacy as LinkedBaseModel + + class T(LinkedBaseModel): + id: str + + class M(LinkedBaseModel): + id: str + literal: str | None = None + link: T | None = F1(None, range="T") + + obj = M(id="x", literal="v", link=T(id="ex:t")) + _ = obj.link + return obj, M, "link", T(id="ex:t2") + + +def build_shipped_v2(): + from pydantic import Field + + from oold.model import _LinkedBaseModelLegacy as LinkedBaseModel + + class T(LinkedBaseModel): + id: str + + class M(LinkedBaseModel): + id: str + literal: str | None = None + link: T | None = Field(None, json_schema_extra={"range": "T"}) + + obj = M(id="x", literal="v", link=T(id="ex:t")) + _ = obj.link + return obj, M, "link", T(id="ex:t2") + + +def build_auto_implicit(): + """Auto-descriptor, implicit form: annotated field + range keyword.""" + from oold.model._descriptor import LinkedBaseModel, OoldField + + class T(LinkedBaseModel): + id: str + + class M(LinkedBaseModel): + id: str + literal: str | None = None + link: T | None = OoldField(default=None, range="T") + + M.model_rebuild() + obj = M(id="x", literal="v", link=T(id="ex:t")) + _ = obj.link + return obj, M, "link", T(id="ex:t2") + + +def build_auto_explicit(): + """Auto-descriptor, explicit form: descriptor declared in the class body.""" + from oold.model._descriptor import Link, LinkedBaseModel + + class T(LinkedBaseModel): + id: str + + class M(LinkedBaseModel): + id: str + literal: str | None = None + link = Link(T) + + obj = M(id="x", literal="v", link=T(id="ex:t")) + _ = obj.link + return obj, M, "link", T(id="ex:t2") + + +def build_ref(): + """Explicit Ref[T] wrapper.""" + from oold.model._ref import OoldModel, Ref + + class T(OoldModel): + id: str + + class M(OoldModel): + id: str + literal: str | None = None + link: Ref[T] | None = None + + obj = M(id="x", literal="v", link=T(id="ex:t")) + _ = obj.link + return obj, M, "link", T(id="ex:t2") + + +VARIANTS = { + "plain_v1": ("plain pydantic v1", build_plain_v1), + "plain_v2": ("plain pydantic v2", build_plain_v2), + "gated": ("gated __getattribute__", build_gated), + "shipped_v1": ("legacy binding v1 ", build_shipped_v1), + "shipped_v2": ("legacy binding v2 ", build_shipped_v2), + "auto_implicit": ("auto-descriptor (implicit)", build_auto_implicit), + "auto_explicit": ("auto-descriptor (explicit)", build_auto_explicit), + "ref": ("explicit Ref[T]", build_ref), +} + +OPS = ["plain_read", "plain_write", "link_read", "link_write", "query"] + + +def measure(key: str) -> dict: + obj, cls, linkname, linkval = VARIANTS[key][1]() + out = {} + out["plain_read"] = min(timeit.repeat(lambda: obj.literal, number=N, repeat=REP)) + + def setp(): + obj.literal = "v" + + out["plain_write"] = min(timeit.repeat(setp, number=N, repeat=REP)) + + if linkname: + out["link_read"] = min(timeit.repeat(lambda: getattr(obj, linkname), number=N, repeat=REP)) + + def setl(): + setattr(obj, linkname, linkval) + + try: + out["link_write"] = min(timeit.repeat(setl, number=N, repeat=REP)) + except Exception: + out["link_write"] = None + try: + out["query"] = min(timeit.repeat(lambda: cls.literal == "John", number=N, repeat=REP)) + except Exception: + out["query"] = None + return out + + +def main() -> None: + results = {} + for key in VARIANTS: + proc = subprocess.run([sys.executable, __file__, key], capture_output=True, text=True) + if proc.returncode != 0: + print(f"{key}: FAILED\n{proc.stderr[-500:]}\n") + continue + results[key] = eval(proc.stdout.strip()) # noqa: S307 + + base = results.get("plain_v2", {}).get("plain_read") + print(f"\n{N:,}x per op, best of {REP}, isolated processes") + print("times in ms; (x) = relative to plain pydantic v2 plain-read\n") + head = f"{'variant':28}" + "".join(f"{o:>16}" for o in OPS) + print(head) + print("-" * len(head)) + for key, vals in results.items(): + row = f"{VARIANTS[key][0]:28}" + for op in OPS: + t = vals.get(op) + if t is None: + row += f"{'na':>16}" + else: + row += f"{t * 1e3:8.1f}({t / base:4.1f}x)" + print(row) + + +if __name__ == "__main__": + if len(sys.argv) > 1: + print(repr(measure(sys.argv[1]))) + else: + main() diff --git a/examples/check_binding_features.py b/examples/check_binding_features.py new file mode 100644 index 0000000..617295a --- /dev/null +++ b/examples/check_binding_features.py @@ -0,0 +1,379 @@ +"""Feature-check every graph-object binding variant against the requirements. + +Each requirement is verified by actually exercising the variant, not asserted by +hand. Prints a matrix of ok / FAIL / na per variant, so regressions and gaps are +visible rather than claimed. + +Each variant runs in a **separate subprocess**: importing ``oold.model`` +monkeypatches ``pydantic.fields.FieldInfo`` process-wide, and the monkeypatch +check itself must therefore be isolated. + +Run it: + + python examples/check_binding_features.py +""" + +import subprocess +import sys +from collections.abc import Callable + +REQUIREMENTS = [ + ("syntax_unchanged", "standard annotations, no wrapper type in declaration"), + ("build_by_iri", "construct a link from an IRI string"), + ("build_by_object", "construct a link from a model instance"), + ("lazy", "no backend call before first access"), + ("real_object", "access returns the real target (isinstance holds)"), + ("polymorphic", "resolves to the actual subclass via its type IRI"), + ("batched", "N-item list resolves in ONE backend call"), + ("cached", "second access does not re-resolve"), + ("mutation", "assignment replaces the link and invalidates the cache"), + ("link_validated", "linked object is validated by its own model on construction"), + ("list_lookup", "list IRI lookup: links['ex:t2']"), + ("list_filter", "list filtering: links[T.label == 'two']"), + ("list_projection", "list attribute projection: links.label"), + ("serialize_iri", "serialisation emits IRIs for links"), + ("query_dsl", "Cls.field == v and Cls[cond] work"), + ("typed_extras", "validated json_schema_extra (OoldExtra)"), + ("no_monkeypatch", "import does not patch pydantic FieldInfo"), +] + +VARIANT_NAMES = { + "shipped_v1": "shipped v1", + "shipped_v2": "shipped v2", + "auto_implicit": "auto (implicit)", + "auto_explicit": "auto (explicit)", + "ref": "Ref[T]", +} + +MODULE_OF = { + "shipped_v1": "oold.model.v1", + "shipped_v2": "oold.model", + "auto_implicit": "oold.model._descriptor", + "auto_explicit": "oold.model._descriptor", + "ref": "oold.model._ref", +} + +CALLS: list[list] = [] + +# Stored documents carry a type IRI so polymorphic dispatch can be exercised. +DATA = { + "ex:t1": {"id": "ex:t1", "label": "one", "type": "ex:T"}, + "ex:t2": {"id": "ex:t2", "label": "two", "type": "ex:T"}, + "ex:s1": {"id": "ex:s1", "label": "sub", "type": "ex:S"}, +} + + +def counting_store(): + from oold.backend.document_store import SimpleDictDocumentStore + from oold.backend.interface import SetResolverParam, set_resolver + + class Counting(SimpleDictDocumentStore): + def resolve_iris(self, iris): + CALLS.append(list(iris)) + return super().resolve_iris(iris) + + s = Counting() + s.store_json_dicts(DATA) + set_resolver(SetResolverParam(iri="ex", resolver=s)) + return s + + +class Probe: + """Collects requirement results, isolating failures per check.""" + + def __init__(self) -> None: + self.res: dict[str, bool | None] = {} + + def check(self, name: str, fn: Callable[[], bool]) -> None: + try: + self.res[name] = bool(fn()) + except Exception: + self.res[name] = False + + def set(self, name: str, value: bool | None) -> None: + self.res[name] = value + + +def check_shipped(version: int) -> dict: + p = Probe() + if version == 1: + from pydantic.v1 import Field as F + + from oold.model.v1 import LinkedBaseModel as Base + + def link_field(): + return F(None, range="T") + + else: + from pydantic import Field as F + + from oold.model import LinkedBaseModel as Base + + def link_field(): + return F(None, json_schema_extra={"range": "T"}) + + class T(Base): + id: str + label: str | None = None + type: str | None = "ex:T" + + class S(T): # subclass for the polymorphism probe + type: str | None = "ex:S" + + class M(Base): + id: str + name: str | None = None + links: list[T] | None = link_field() + + p.set("syntax_unchanged", True) # standard annotations, List[T] + counting_store() + + CALLS.clear() + m = M(id="ex:m", links=["ex:t1", "ex:t2"]) + p.set("build_by_iri", True) + p.check("lazy", lambda: len(CALLS) == 0) + got = m.links + p.check("real_object", lambda: isinstance(got[0], T) and got[0].label == "one") + p.check("batched", lambda: len(CALLS) == 1 and len(CALLS[0]) == 2) + before = len(CALLS) + _ = m.links + p.check("cached", lambda: len(CALLS) == before) + p.check( + "build_by_object", + lambda: isinstance(M(id="ex:m2", links=[T(id="ex:t1")]).links[0], T), + ) + p.check("polymorphic", lambda: isinstance(M(id="ex:m3", links=["ex:s1"]).links[0], S)) + + def mutate(): + m.links = [T(id="ex:t2", label="two")] + return m.links[0].id == "ex:t2" + + p.check("mutation", mutate) + + def link_validated(): + # even when the parent's field validation is bypassed, the linked + # object must still be validated by its own model at construction + try: + M(id="ex:mv", links=[{"label": "no id"}]) # 'id' is required on T + return False + except Exception: + return True + + p.check("link_validated", link_validated) + + p.check("list_lookup", lambda: m.links["ex:t2"].id == "ex:t2") + p.check("list_filter", lambda: [x.id for x in m.links[T.label == "two"]] == ["ex:t2"]) + p.check("list_projection", lambda: list(m.links.label) == ["two"]) + p.check("serialize_iri", lambda: m.to_json().get("links") == ["ex:t2"]) + p.check("query_dsl", lambda: getattr(M.name == "John", "field", None) == "name") + p.set("typed_extras", False) # raw dict only + return p.res + + +def check_auto(explicit: bool) -> dict: + from oold.model._descriptor import ( + LinkedBaseModel, + LinkList, + OoldExtra, + OoldField, + ) + + p = Probe() + + class T(LinkedBaseModel): + id: str + label: str | None = None + type: str | None = "ex:T" + + class S(T): + type: str | None = "ex:S" + + if explicit: + + class M(LinkedBaseModel): + id: str + name: str | None = None + links = LinkList(T) + + p.set("syntax_unchanged", False) # unannotated descriptor assignment + else: + + class M(LinkedBaseModel): + id: str + name: str | None = None + links: list[T] | None = OoldField(default=None, range="T") + + p.set("syntax_unchanged", True) + + counting_store() + CALLS.clear() + m = M(id="ex:m", links=["ex:t1", "ex:t2"]) + p.set("build_by_iri", True) + p.check("lazy", lambda: len(CALLS) == 0) + got = m.links + p.check("real_object", lambda: isinstance(got[0], T) and got[0].label == "one") + p.check("batched", lambda: len(CALLS) == 1 and len(CALLS[0]) == 2) + before = len(CALLS) + _ = m.links + p.check("cached", lambda: len(CALLS) == before) + p.check( + "build_by_object", + lambda: isinstance(M(id="ex:m2", links=[T(id="ex:t1")]).links[0], T), + ) + # prototype constructs the declared target, it does not dispatch on type IRI + p.check("polymorphic", lambda: isinstance(M(id="ex:m3", links=["ex:s1"]).links[0], S)) + + def mutate(): + m.links = [T(id="ex:t2", label="two")] + return m.links[0].id == "ex:t2" + + p.check("mutation", mutate) + + def link_validated(): + # even when the parent's field validation is bypassed, the linked + # object must still be validated by its own model at construction + try: + M(id="ex:mv", links=[{"label": "no id"}]) # 'id' is required on T + return False + except Exception: + return True + + p.check("link_validated", link_validated) + + p.check("list_lookup", lambda: m.links["ex:t2"].id == "ex:t2") + p.check("list_filter", lambda: [x.id for x in m.links[T.label == "two"]] == ["ex:t2"]) + p.check("list_projection", lambda: list(m.links.label) == ["two"]) + p.check( + "serialize_iri", + lambda: m.model_dump(exclude_none=True).get("links") == ["ex:t2"], + ) + + def query(): + cond = M.name == "John" + return getattr(cond, "field", None) == "name" and M[cond] is not None + + p.check("query_dsl", query) + + def typed(): + try: + OoldExtra(range="") + return False + except Exception: + return True + + p.check("typed_extras", typed) + return p.res + + +def check_ref() -> dict: + from oold.model._ref import OoldModel, Ref + + p = Probe() + + class T(OoldModel): + id: str + label: str | None = None + type: str | None = "ex:T" + + class M(OoldModel): + id: str + name: str | None = None + links: list[Ref[T]] | None = None + + p.set("syntax_unchanged", False) # Ref[T] wrapper appears in the annotation + counting_store() + CALLS.clear() + m = M(id="ex:m", links=["ex:t1", "ex:t2"]) + p.set("build_by_iri", True) + p.check("lazy", lambda: len(CALLS) == 0) + got = m.links + p.check("real_object", lambda: isinstance(got[0], T)) # it is a Ref, expected False + _ = [r.label for r in got] + p.check("batched", lambda: len(CALLS) == 1 and len(CALLS[0]) == 2) + before = len(CALLS) + _ = [r.label for r in m.links] + p.check("cached", lambda: len(CALLS) == before) + p.check("build_by_object", lambda: M(id="ex:m2", links=[T(id="ex:t1")]) is not None) + p.check("polymorphic", lambda: False) + + def mutate(): + m.links = [T(id="ex:t2", label="two")] + return m.model_dump(exclude_none=True).get("links") == ["ex:t2"] + + p.check("mutation", mutate) + + def link_validated(): + # even when the parent's field validation is bypassed, the linked + # object must still be validated by its own model at construction + try: + M(id="ex:mv", links=[{"label": "no id"}]) # 'id' is required on T + return False + except Exception: + return True + + p.check("link_validated", link_validated) + + p.check("list_lookup", lambda: m.links["ex:t2"].id == "ex:t2") + p.check("list_filter", lambda: [x.id for x in m.links[T.label == "two"]] == ["ex:t2"]) + p.check("list_projection", lambda: list(m.links.label) == ["two"]) + p.check( + "serialize_iri", + lambda: M(id="ex:m4", links=["ex:t2"]).model_dump(exclude_none=True).get("links") == ["ex:t2"], + ) + p.check("query_dsl", lambda: False) + p.set("typed_extras", False) + return p.res + + +def monkeypatch_check(module: str) -> bool: + code = f"import pydantic.fields as pf; import {module}; print(pf.FieldInfo.__name__)" + out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + return out.returncode == 0 and out.stdout.strip() == "FieldInfo" + + +def run(key: str) -> dict: + if key == "shipped_v1": + res = check_shipped(1) + elif key == "shipped_v2": + res = check_shipped(2) + elif key == "auto_implicit": + res = check_auto(explicit=False) + elif key == "auto_explicit": + res = check_auto(explicit=True) + elif key == "ref": + res = check_ref() + else: + raise KeyError(key) + res["no_monkeypatch"] = monkeypatch_check(MODULE_OF[key]) + return res + + +def main() -> None: + results = {} + for key in VARIANT_NAMES: + proc = subprocess.run([sys.executable, __file__, key], capture_output=True, text=True) + if proc.returncode != 0: + print(f"{key}: ERROR\n{proc.stderr[-700:]}\n") + continue + results[key] = eval(proc.stdout.strip()) # noqa: S307 + + width = max(len(r) for r, _ in REQUIREMENTS) + 2 + header = f"{'requirement':{width}}" + "".join(f"{VARIANT_NAMES[k]:>17}" for k in results) + print(header) + print("-" * len(header)) + for req, _desc in REQUIREMENTS: + row = f"{req:{width}}" + for key in results: + v = results[key].get(req) + row += f"{('ok' if v else 'FAIL') if v is not None else 'na':>17}" + print(row) + print("\nlegend") + for req, desc in REQUIREMENTS: + print(f" {req:18} {desc}") + + +if __name__ == "__main__": + if len(sys.argv) > 1: + print(repr(run(sys.argv[1]))) + else: + main() diff --git a/examples/notation_example.py b/examples/notation_example.py new file mode 100644 index 0000000..9c34718 --- /dev/null +++ b/examples/notation_example.py @@ -0,0 +1,173 @@ +"""Smoke test for the reviewed link declaration notations. + +Runnable end-to-end and self-checking, so it doubles as a copy-paste starting +point. Covers the notations discussed in oold-python#107: + +1. ``OoldField()`` with no arguments - the link target is inferred from the + annotation, so the schema IRI is not repeated in Python. +2. ``Link[T]`` / ``LinkList[T]`` as the whole annotation. A link has two types - + you read a resolved object, you may write that object *or* a reference to it - + and these carry both, so a type checker accepts an IRI on construction and + still narrows the read to the target type. +3. **Union arms** mixing a literal, an inline object and a reference. + +Run it: + + python examples/notation_example.py + +Generated code keeps the unchanged declaration syntax of +``oold.model._descriptor``; ``oold.model._notation`` adds the notations above on +top of it. Which notation supports what is tabulated in +``docs/design/graph-object-binding.md`` section 3.3. +""" + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model import Link, LinkList, OoldField +from oold.model._notation import OoldModel + + +class Organization(OoldModel): + id: str + name: str | None = None + type: str | None = "ex:Organization" + + +class Location(OoldModel): + id: str | None = None # optional: an inline value may be a blank node + address: str | None = None + type: str | None = "ex:Location" + + +class Person(OoldModel): + id: str + name: str | None = None + type: str | None = "ex:Person" + + # 1. Link[T] / LinkList[T] as the whole annotation - the recommended form. + # Reads give the resolved object, writes accept an object, an IRI or a + # JSON object, and a type checker sees both (see the coverage note below). + knows: LinkList["Person"] = OoldField() + employer: Link[Organization] = OoldField() + + # 2. the plain form: identical at runtime, no range= needed either, but a + # type checker only sees list[Person] and so rejects a list of IRIs + friends: list["Person"] = OoldField() + + # 3. union: literal text | inline object | reference + location: str | Location | None = OoldField(link=True) + + +Person.model_rebuild() + + +def setup_backend() -> SimpleDictDocumentStore: + store = SimpleDictDocumentStore() + store.store_json_dicts({ + "ex:bob": {"id": "ex:bob", "name": "Bob", "type": "ex:Person"}, + "ex:carol": {"id": "ex:carol", "name": "Carol", "type": "ex:Person"}, + "ex:acme": {"id": "ex:acme", "name": "ACME", "type": "ex:Organization"}, + "ex:eiffel": { + "id": "ex:eiffel", + "address": "Champ de Mars", + "type": "ex:Location", + }, + }) + set_resolver(SetResolverParam(iri="ex", resolver=store)) + return store + + +def main() -> None: + setup_backend() + + alice = Person( + id="ex:alice", + name="Alice", + knows=["ex:bob", "ex:carol"], # by IRI, resolved on demand + employer="ex:acme", + friends=[Person(id="ex:bob", name="Bob")], # or by object + ) + + print("1. Link[T] / LinkList[T] - target inferred, and statically typed") + assert alice.knows[0].name == "Bob" + assert isinstance(alice.knows[0], Person) # a real Person, not a proxy + print(" knows[0].name =", alice.knows[0].name) + print(" isinstance(.., Person) =", isinstance(alice.knows[0], Person)) + + print("\n2. the plain list[T] form - identical at runtime") + assert isinstance(alice.employer, Organization) + assert alice.employer.name == "ACME" + assert isinstance(alice.friends[0], Person) + print(" employer.name =", alice.employer.name) + print(" friends[0].name =", alice.friends[0].name) + + print("\n3. union arms: text | inline object | reference") + text = Person(id="ex:p-text", location="at the Eiffel Tower") + ref = Person(id="ex:p-ref", location={"@id": "ex:eiffel"}) + inline = Person( + id="ex:p-inline", + location={"id": "ex:office", "address": "Main St 1", "type": "ex:Location"}, + ) + blank = Person(id="ex:p-blank", location={"address": "no id", "type": "ex:Location"}) + + # A union field is str | Location | None, so narrow it to a local before + # dereferencing - the same hygiene any union needs, and what lets a type + # checker follow along. + ref_loc, inline_loc, blank_loc = ref.location, inline.location, blank.location + assert text.location == "at the Eiffel Tower" # stays a literal + assert isinstance(ref_loc, Location) # resolved reference + assert isinstance(inline_loc, Location) and isinstance(blank_loc, Location) + assert ref_loc.address == "Champ de Mars" + assert inline_loc.address == "Main St 1" + assert blank.link_iris("location") is None # no IRI -> blank node + print(" text ->", repr(text.location)) + print(" ref ->", ref_loc.address) + print(" inline ->", inline_loc.address) + print(" blank ->", blank_loc.address, "(no IRI)") + + print("\n4. serialisation: links to IRIs; references boxed where a literal arm exists") + dumped = alice.model_dump(exclude_none=True) + assert dumped["knows"] == ["ex:bob", "ex:carol"] + assert dumped["employer"] == "ex:acme" + # boxed as {"@id": ...} because the field also accepts a literal + assert ref.model_dump(exclude_none=True)["location"] == {"@id": "ex:eiffel"} + assert isinstance(blank.model_dump(exclude_none=True)["location"], dict) + print(" alice ->", dumped) + print(" ref ->", ref.model_dump(exclude_none=True)) + print(" blank ->", blank.model_dump(exclude_none=True)) + + print("\n5. lazy resolution and query DSL") + lazy = Person(id="ex:lazy", knows=["ex:bob"]) + assert lazy.link_iris("knows") == ["ex:bob"] # inspect without resolving + # The class-level DSL builds a Condition at runtime, but a type checker + # sees BaseModel.__eq__ and reads this as bool. The subscript overloads + # accept bool for that reason, so Person[cond] keeps its result type. + condition = Person.name == "Bob" + assert condition.field == "name" + print(" link_iris('knows') =", lazy.link_iris("knows")) + print(" Person.name == 'Bob' =", condition) + + print("\n6. de-serialisation: every union arm survives a round trip") + for label, value, expected in [ + ("text ", "at the Eiffel Tower", str), + ("reference", {"@id": "ex:eiffel"}, Location), + ("inline ", {"address": "Main St 1", "type": "ex:Location"}, Location), + ]: + original = Person(id="ex:rt", location=value) + payload = original.model_dump(exclude_none=True) + restored = Person(**payload) + assert isinstance(restored.location, expected), label + shown = str(payload["location"])[:34] + print(f" {label} {shown:36} -> {type(restored.location).__name__}") + + restored = Person(**alice.model_dump(exclude_none=True)) + assert [x.id for x in restored.knows] == ["ex:bob", "ex:carol"] + restored_employer = restored.employer # to-one link: narrow before use + assert restored_employer is not None and restored_employer.name == "ACME" + print(" lists and to-one links round trip too") + + print("\nALL CHECKS PASSED") + + +if __name__ == "__main__": + main() diff --git a/examples/wiki_data.py b/examples/wiki_data.py index b2acca2..458220b 100644 --- a/examples/wiki_data.py +++ b/examples/wiki_data.py @@ -1,17 +1,32 @@ -from typing import Optional +"""Resolve Wikidata entities as typed objects over a public SPARQL endpoint. -from pydantic import ConfigDict, Field +Shows the binding against a backend nobody controls: the classes below declare +only a JSON-LD context and which properties are links, and +``Person["Item:Q80"]`` turns an IRI into a ``Person`` whose ``father`` is another +``Person``, fetched on first access. + +Run it: + + python examples/wiki_data.py + +Two details are specific to Wikidata: + +* the class IRI is the **expanded** entity IRI, because that is what arrives in + ``@type``; the registry matches type IRIs literally, without prefix expansion; +* the resolver rewrites ``wdt:P31`` (instance of) into ``@type``, so the context + aliases ``type`` to ``@type`` rather than mapping it to P31. +""" + +from pydantic import ConfigDict from oold.backend.interface import SetResolverParam, set_resolver from oold.backend.sparql import WikiDataSparqlResolver # based on pydantic v2 -from oold.model import LinkedBaseModel - +from oold.model import Link, LinkedBaseModel, LinkNotResolved, OoldField -class MultiLanguageString(LinkedBaseModel): - text: str - lang: str +WD_ENTITY = "http://www.wikidata.org/entity/" +ENTITY_SCHEMA = "https://oo-ld.org/examples/wikidata/Entity" class WikiDataEntity(LinkedBaseModel): @@ -20,84 +35,107 @@ class WikiDataEntity(LinkedBaseModel): "@context": { # aliases "id": "@id", + "type": "@type", # prefixes "p": "http://www.wikidata.org/prop/", "wdt": "http://www.wikidata.org/prop/direct/", - "Item": "http://www.wikidata.org/entity/", - "type": "wdt:P31", - "name": { - "@id": "wdt:P373", - "@type": "http://www.w3.org/2001/XMLSchema#string", - }, - # "label": { - # "@id": "http://www.w3.org/2000/01/rdf-schema#label", - # "@container": "@set", - # "@context": { - # "text": "@value", - # "lang": "@language", - # } - # }, + "Item": WD_ENTITY, + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + # rdfs:label is language-tagged and multi-valued; scoping the + # term to one language makes it compact to a plain string. + # wdt:P373 (Commons category) would read more directly but is + # sparse - most entities do not carry one. + "name": {"@id": "rdfs:label", "@language": "en"}, }, - "iri": "Entity.json", # the IRI of the schema + "iri": ENTITY_SCHEMA, # the IRI of the schema } ) id: str - type: str | None - # label: Optional[List[MultiLanguageString]] = None + type: str | None = None name: str | None = None - @classmethod - def get_class_iri(cls): - # return default value of field 'type' if not set - if cls.model_fields.get("type") and cls.model_fields["type"].default is not None: - return cls.model_fields["type"].default - def get_iri(self): - return "ex:" + self.name + return self.id class Person(WikiDataEntity): model_config = ConfigDict( json_schema_extra={ "@context": [ - "Entity.json", # import the context of the parent class + ENTITY_SCHEMA, # import the context of the parent class { - # object property definition + # object property pointing to another Person "father": { "@id": "wdt:P22", "@type": "@id", }, - "knows": { - "@id": "schema:knows", - "@type": "@id", - "@container": "@set", - }, }, ], - "iri": "Q5", + # The class IRI has to be the expanded form: it is compared with the + # @type of the incoming document, and Q5 is "human". + "iri": WD_ENTITY + "Q5", } ) - type: str | None = "wd:Q5" # Q5 is the Wikidata item for human - father: Optional["Person"] = Field( - None, - json_schema_extra={"range": "Person.json"}, + type: str | None = "Item:Q5" + # Link[T] rather than "Person | None": the annotation says what *reading* + # the link yields, so a chain can be written plainly and guarded once. + # Ancestry does run out - that is what the try/except in main() is for. + father: Link["Person"] = OoldField() + + +Person.model_rebuild() + +# Wikidata attributes requests by user agent and throttles the ones it cannot +# place - the SPARQLWrapper default is answered with "429 Aggressively +# rate-limiting to 1 req / min". The resolver sends a descriptive one by default. +set_resolver( + SetResolverParam( + iri="Item", + resolver=WikiDataSparqlResolver(endpoint="https://query.wikidata.org/sparql"), ) - knows: list["Person"] | None = Field( - None, - # object property pointing to another Person - json_schema_extra={"range": "Person.json"}, - ) - - -# create a resolver to resolve IRIs to objects - - -r = WikiDataSparqlResolver(endpoint="https://query.wikidata.org/sparql") -set_resolver(SetResolverParam(iri="Item", resolver=r)) - -# Example usage: -p = Person["Item:Q80"] # Douglas Adams -print(p.model_dump_json(indent=2)) -print(p) -print(p.father) -print(p.father.father) +) + + +def main() -> None: + person = Person["Item:Q80"] # Tim Berners-Lee + assert person is not None, "Q80 not resolved - the endpoint may be unavailable" + print("resolved:", person.id) + print("name: ", person.name) + print("type: ", person.type) + + # the link is an IRI in the payload and a Person once read + print("\nfather is fetched on access, not on construction") + print(" stored IRI:", person.get_iri_ref("father")) + father = person.father + assert isinstance(father, Person), type(father) + print(" resolved: ", father.id, "-", father.name) + + print("\nplain chaining - no guard, no narrowing, no cast") + ggf = person.father.father.father + print(" great-grandfather:", ggf.name) + + print("\nthe same walk, until the data runs out") + ancestor, generations = person, 0 + try: + while True: + ancestor = ancestor.father + generations += 1 + print(f" {generations} generation(s) back:", ancestor.name) + except LinkNotResolved: + # Ancestry runs out. Declaring the link mandatory is what turns that + # into one exception at the end rather than a guard at every hop. + print(f" no father recorded for {ancestor.name} - walked {generations} generation(s)") + + print("\nquery: the same DSL, translated to SPARQL by the resolver") + found = Person[Person.name == "Tim Berners-Lee"] + print(" Person[Person.name == 'Tim Berners-Lee'] ->", [p.id for p in found or []]) + + print("\nserialisation writes the link back as an IRI") + dumped = person.to_json() + print(" father ->", dumped["father"]) + + print("\nALL CHECKS PASSED") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 23cb9c5..f4bd857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,12 @@ changelog_file = "CHANGELOG.md" [tool.ty.environment] python = "./.venv" python-version = "3.10" +# src layout: without this ty cannot resolve `oold.*` from tests, and every +# import there silently becomes Unknown - which makes those checks vacuous rather +# than failing, so it is not visible in the output. Note that an incomplete +# ./.venv has the same effect on third-party imports, and an unresolved pydantic +# then reports a spurious conflicting-metaclass on every model. +extra-paths = ["src"] [tool.ty.src] # Excluded from type checking: @@ -134,12 +140,14 @@ python-version = "3.10" # - src/oold/drafts: draft/experimental code (also excluded from ruff) # - src/oold/ui: optional UI integrations that import optional, un-installed # packages (panel, anywidget, nicegui, traitlets, ...) -# - examples: illustrative scripts, not part of the distributed package +# +# `examples` is checked: they are the documented way to declare a link, and +# excluding them let a broken import and the wrong static type for +# `Model[Model.field == x]` sit there unnoticed. exclude = [ "tests/data", "src/oold/drafts", "src/oold/ui", - "examples", ] [tool.ty.rules] @@ -150,6 +158,10 @@ exclude = [ # they are downgraded to keep `ty check` meaningful without mass inline ignores. unresolved-attribute = "ignore" not-subscriptable = "ignore" +# Assigning an IRI to a link field is no longer a reason for this one: the +# Link[T] / LinkList[T] annotations type both directions, and the override below +# re-enables the rule for the file that asserts that contract. What remains are +# unrelated sites (json_tools, the codegen spike) not yet cleaned up. invalid-argument-type = "ignore" invalid-assignment = "ignore" unknown-argument = "ignore" @@ -185,6 +197,9 @@ traitlets = "traitlets" nicegui = "nicegui" [tool.deptry.per_rule_ignores] +# stdlib from 3.14 (PEP 649); deptry resolves against the running interpreter, +# which is older, so it reads the version-gated import as a missing dependency. +DEP001 = ["annotationlib"] # Declared in UI extras and required at runtime by the panel/jupyter # integrations, but not imported directly from `src`. DEP002 = ["jupyter_bokeh", "ipykernel"] @@ -261,7 +276,10 @@ ignore = [ # Tests and examples use literal credentials/tokens and broad exception asserts # on purpose; the bandit/bugbear security rules are noise there. "tests/*" = ["S101", "S105", "S106", "B017"] -"examples/*" = ["S105", "S106"] +# Examples and spikes are runnable demos: they self-check with `assert` and +# re-invoke themselves via subprocess to isolate measurements. +"examples/*" = ["S101", "S105", "S106", "S603", "TRY300"] +"src/oold/experimental/*" = ["S101"] [tool.ruff.format] preview = true @@ -272,3 +290,15 @@ skip_empty = true [tool.coverage.run] branch = true source = ["src"] + +[[tool.ty.overrides]] +# The static contract of the link and query API is asserted here, so the rules +# it exercises must actually run - the repo-wide downgrades above would +# otherwise make every assert_type in it vacuous. +include = ["tests/typing/**"] + +[tool.ty.overrides.rules] +invalid-argument-type = "error" +invalid-assignment = "error" +unresolved-attribute = "error" +not-subscriptable = "error" diff --git a/src/oold/backend/document_store.py b/src/oold/backend/document_store.py index dd74d14..7746c21 100644 --- a/src/oold/backend/document_store.py +++ b/src/oold/backend/document_store.py @@ -81,7 +81,6 @@ def _query( context: dict | None = None, data: dict[str, dict] | None = None, ) -> set[str]: - print("QUERY", query) if data is None: data = self._store if isinstance(query, Condition): diff --git a/src/oold/backend/sparql.py b/src/oold/backend/sparql.py index ddff6c3..052f690 100644 --- a/src/oold/backend/sparql.py +++ b/src/oold/backend/sparql.py @@ -1,11 +1,33 @@ import json +from typing import Any from pydantic import ConfigDict from rdflib import Graph -from SPARQLWrapper import JSONLD, SPARQLWrapper +from SPARQLWrapper import JSON, JSONLD, SPARQLWrapper from oold.backend.auth import UserPwdCredential, get_credential -from oold.backend.interface import Backend, Resolver, StoreResult +from oold.backend.interface import ( + Backend, + ComparisonOperator, + Query, + QueryParam, + ResolveParam, + Resolver, + ResolveResult, + StoreResult, +) + +WD_INSTANCE_OF = "http://www.wikidata.org/prop/direct/P31" +"""Wikidata's "instance of" - what this module maps to ``@type``.""" + +DEFAULT_USER_AGENT = "oold-python (https://github.com/OO-LD/oold-python)" +"""Sent with every SPARQL request. + +Public endpoints identify clients by user agent and throttle the ones they +cannot attribute: Wikidata answers the SPARQLWrapper default with +``429 Aggressively rate-limiting to 1 req / min``, so a request that looks +correct still fails. Their policy asks for a tool name and a contact URL. +""" class LocalSparqlResolver(Resolver): @@ -18,6 +40,20 @@ def __init__(self, **kwargs): if self.graph is None: self.graph = Graph() + def query(self, param: QueryParam) -> ResolveResult: + """Same translation as the remote resolver, run against the local graph. + + Having both go through ``_translate`` is what makes the offline test a + check on the translation rather than on a second implementation of it. + """ + model_cls = param.model_cls or self.model_cls + if model_cls is None: + raise ValueError("No model_cls provided in request or resolver") + patterns = _translate(param.query, model_cls, [0]) + rows = self.graph.query("SELECT DISTINCT ?s WHERE {\n" + patterns + "\n}") + iris = [str(row[0]) for row in rows] + return self.resolve(ResolveParam(iris=iris, model_cls=model_cls)) + def resolve_iris(self, iris: list[str]) -> dict[str, dict]: # sparql query to get a node by IRI with all its properties # using CONSTRUCT to get the full node @@ -29,18 +65,7 @@ def resolve_iris(self, iris: list[str]) -> dict[str, dict]: if iri.startswith("http"): iri_filter = f"FILTER (?s = <{iri}>)" # todo: build full iri / prefix mapping from model context - qres = self.graph.query( - """ - PREFIX ex: - CONSTRUCT { - ?s ?p ?o . - } - WHERE { - ?s ?p ?o . - {{{iri_filter}}} - } - """.replace("{{{iri_filter}}}", iri_filter) - ) + qres = self.graph.query(_construct_node(iri_filter, "PREFIX ex: ")) jsonld_dict = json.loads(qres.serialize(format="json-ld"))[0] jsonld_dicts[iri] = jsonld_dict return jsonld_dicts @@ -67,19 +92,53 @@ def store_jsonld_dicts(self, jsonld_dicts: dict[str, dict]) -> StoreResult: self.graph += g return StoreResult(success=True) - def query(): - raise NotImplementedError() - class SparqlResolver(Resolver): model_config = ConfigDict(arbitrary_types_allowed=True) endpoint: str + user_agent: str = DEFAULT_USER_AGENT + query_limit: int = 100 + use_credentials: bool = True + """Whether to look a stored credential up for this endpoint. + + ``find_credential`` matches by substring, so a public endpoint would send + any credential whose key happens to be contained in its URL. + """ + + def _prefixes(self) -> str: + """PREFIX declarations for the generated queries.""" + return "PREFIX ex: " + + def _subject_patterns(self, model_cls: Any) -> str: + """Extra graph patterns constraining ?s. Empty unless a subclass adds any.""" + return "" + + def _post_process(self, jsonld_dict: dict) -> dict: + """Endpoint-specific rewriting of a fetched document.""" + return jsonld_dict def __init__(self, **kwargs): super().__init__(**kwargs) - self._sparql = SPARQLWrapper(self.endpoint) + self._sparql = SPARQLWrapper(self.endpoint, agent=self.user_agent) + + def query(self, param: QueryParam) -> ResolveResult: + """Find the subjects matching a Condition / Query, then resolve them. + + Only the comparison operators the DSL already builds are translated + (eq, ne, lt, le, gt, ge) plus ``&``. Anything else raises rather than + quietly returning the wrong rows. + """ + model_cls = param.model_cls or self.model_cls + if model_cls is None: + raise ValueError("No model_cls provided in request or resolver") + patterns = self._subject_patterns(model_cls) + _translate(param.query, model_cls, [0]) + self._sparql.setQuery("SELECT DISTINCT ?s WHERE {\n" + patterns + "\n} LIMIT " + str(self.query_limit)) + self._sparql.setReturnFormat(JSON) + rows = self._sparql.query().convert()["results"]["bindings"] + iris = [row["s"]["value"] for row in rows] + return self.resolve(ResolveParam(iris=iris, model_cls=model_cls)) def resolve_iris(self, iris: list[str]) -> dict[str, dict]: # sparql query to get a node by IRI with all its properties @@ -88,10 +147,12 @@ def resolve_iris(self, iris: list[str]) -> dict[str, dict]: jsonld_dicts = {} # lookup credential for the endpoint - try: - cred = get_credential(self.endpoint) - except ValueError: - cred = None + cred = None + if self.use_credentials: + try: + cred = get_credential(self.endpoint) + except ValueError: + cred = None if cred is not None and isinstance(cred, UserPwdCredential): self._sparql.setCredentials(cred.username, cred.password.get_secret_value()) @@ -100,68 +161,121 @@ def resolve_iris(self, iris: list[str]) -> dict[str, dict]: # check if the iri is a full IRI or a prefix if iri.startswith("http"): iri_filter = f"FILTER (?s = <{iri}>)" - self._sparql.setQuery( - """ - PREFIX ex: - CONSTRUCT { - ?s ?p ?o . - } - WHERE { - ?s ?p ?o . - {{{iri_filter}}} - } - """.replace("{{{iri_filter}}}", iri_filter) - ) + self._sparql.setQuery(_construct_node(iri_filter, self._prefixes())) self._sparql.setReturnFormat(JSONLD) result: Graph = self._sparql.query().convert() if len(result) == 0: jsonld_dicts[iri] = None continue - jsonld_dict = json.loads(result.serialize(format="json-ld"))[0] - jsonld_dicts[iri] = jsonld_dict + jsonld_dicts[iri] = self._post_process(json.loads(result.serialize(format="json-ld"))[0]) return jsonld_dicts -class WikiDataSparqlResolver(Resolver): - model_config = ConfigDict(arbitrary_types_allowed=True) +class WikiDataSparqlResolver(SparqlResolver): + """Wikidata, which differs from a plain SPARQL endpoint in two ways. - endpoint: str + It states class membership with ``wdt:P31`` rather than ``rdf:type``, so that + is mapped to ``@type`` on the way in and used to constrain queries on the way + out. Everything else - the user agent, the CONSTRUCT, the DSL translation - + is the base resolver's. + """ - def __init__(self, **kwargs): - super().__init__(**kwargs) + endpoint: str = "https://query.wikidata.org/sparql" + use_credentials: bool = False # public endpoint; never send stored secrets - self._sparql = SPARQLWrapper(self.endpoint) + def _prefixes(self) -> str: + return "PREFIX ex: \nPREFIX Item: " - def resolve_iris(self, iris: list[str]) -> dict[str, dict]: - # sparql query to get a node by IRI with all its properties - # using CONSTRUCT to get the full node - # format the result as json-ld - jsonld_dicts = {} - for iri in iris: - iri_filter = f"FILTER (?s = {iri})" - # check if the iri is a full IRI or a prefix - if iri.startswith("http"): - iri_filter = f"FILTER (?s = <{iri}>)" - self._sparql.setQuery( - """ - PREFIX ex: - PREFIX Item: - CONSTRUCT { - ?s ?p ?o . - } - WHERE { - ?s ?p ?o . - {{{iri_filter}}} - } - """.replace("{{{iri_filter}}}", iri_filter) - ) - self._sparql.setReturnFormat(JSONLD) - result: Graph = self._sparql.query().convert() - jsonld_dict = json.loads(result.serialize(format="json-ld"))[0] - # replace http://www.wikidata.org/prop/direct/P31 with @type - if "http://www.wikidata.org/prop/direct/P31" in jsonld_dict: - jsonld_dict["@type"] = jsonld_dict.pop("http://www.wikidata.org/prop/direct/P31")[0]["@id"] - jsonld_dicts[iri] = jsonld_dict + def _subject_patterns(self, model_cls: Any) -> str: + # A label matches far more than one kind of thing - "Tim Berners-Lee" is + # also a book edition - and resolving those fails on an unknown type IRI. + class_iri = next((iri for iri in _as_list(model_cls.get_cls_iri()) if str(iri).startswith("http")), None) + if not class_iri: + return "" + return f" ?s <{WD_INSTANCE_OF}> <{class_iri}> .\n" - return jsonld_dicts + def _post_process(self, jsonld_dict: dict) -> dict: + if WD_INSTANCE_OF in jsonld_dict: + jsonld_dict["@type"] = jsonld_dict.pop(WD_INSTANCE_OF)[0]["@id"] + return jsonld_dict + + +_SPARQL_OPERATORS = { + ComparisonOperator.EQ: "=", + ComparisonOperator.NE: "!=", + ComparisonOperator.LT: "<", + ComparisonOperator.LE: "<=", + ComparisonOperator.GT: ">", + ComparisonOperator.GE: ">=", +} + + +def _construct_node(iri_filter: str, prefixes: str = "") -> str: + """CONSTRUCT every triple of one subject. Shared by all three resolvers.""" + return f""" + {prefixes} + CONSTRUCT {{ ?s ?p ?o . }} + WHERE {{ + ?s ?p ?o . + {iri_filter} + }} + """ + + +def _as_list(value) -> list: + if value is None: + return [] + return value if isinstance(value, list) else [value] + + +def _expand_term(model_cls, field: str, value) -> tuple[str, str]: + """Return (predicate IRI, SPARQL literal) for a field of ``model_cls``. + + Both come from expanding a probe document against the model's own JSON-LD + context, so the term definition decides the predicate *and* the literal form + - a term scoped to ``@language: en`` yields ``"x"@en``, one with an + ``@type`` yields ``"x"^^``. Re-deriving either by hand would be a + second, divergent reading of the context. + """ + from pydantic import BaseModel as _BaseModel + from pyld import jsonld as _jsonld + + from oold.static import build_context, get_jsonld_context_loader + + context = build_context(model_cls, _BaseModel) + _jsonld.set_document_loader(get_jsonld_context_loader(model_cls, _BaseModel)) + expanded = _jsonld.expand({"@context": context, field: value}) + if not expanded: + raise ValueError(f"{model_cls.__name__}.{field} is not mapped by the model context") + node = expanded[0] + predicate = next((key for key in node if not key.startswith("@")), None) + if predicate is None: + raise ValueError(f"{model_cls.__name__}.{field} is not mapped by the model context") + entry = node[predicate][0] + if "@id" in entry: + return predicate, f"<{entry['@id']}>" + literal = json.dumps(str(entry["@value"])) + if entry.get("@language"): + return predicate, f"{literal}@{entry['@language']}" + if entry.get("@type"): + return predicate, f"{literal}^^<{entry['@type']}>" + if isinstance(value, (bool, int, float)): + return predicate, json.dumps(value) + return predicate, literal + + +def _translate(node, model_cls, counter: list[int]) -> str: + """Render a Condition or Query as SPARQL graph patterns.""" + if isinstance(node, Query): + if node.operator != "and": + raise NotImplementedError(f"Unsupported query operator: {node.operator}") + return _translate(node.op1, model_cls, counter) + "\n" + _translate(node.op2, model_cls, counter) + predicate, literal = _expand_term(model_cls, node.field, node.value) + operator = node.operator or ComparisonOperator.EQ + if operator == ComparisonOperator.EQ: + # a plain pattern is both selective and index-friendly + return f" ?s <{predicate}> {literal} ." + counter[0] += 1 + var = f"?v{counter[0]}" + return f" ?s <{predicate}> {var} .\n FILTER({var} {_SPARQL_OPERATORS[operator]} {literal})" diff --git a/src/oold/experimental/__init__.py b/src/oold/experimental/__init__.py new file mode 100644 index 0000000..d6ea6f0 --- /dev/null +++ b/src/oold/experimental/__init__.py @@ -0,0 +1,8 @@ +"""Experimental, non-shipping prototypes. + +Modules here validate design directions (see +``docs/design/graph-object-binding.md``) without touching the shipped +``oold.model``. They are intentionally isolated: importing this package must +not trigger the process-wide ``pydantic.fields.FieldInfo`` monkeypatch that +``oold.model`` performs. +""" diff --git a/src/oold/experimental/codegen_spike.py b/src/oold/experimental/codegen_spike.py new file mode 100644 index 0000000..47446c8 --- /dev/null +++ b/src/oold/experimental/codegen_spike.py @@ -0,0 +1,308 @@ +"""Spike: IR-based code generation for OO-LD schemas. + +Companion to ``docs/design/graph-object-binding.md`` section 0c ("own generator +vs datamodel-code-generator"). It shows that the three structural problems the +current toolchain fights with regex on generated *text* - + +1. multiple ``allOf`` inheritance (``osw-python-package-generator``'s + ``_fix_missing_allof_bases``), +2. class reuse / dedup by ``x-oold-uuid`` (the UUID-dedup passes in + ``replace_duplicated_classes_with_imports``), +3. typed ``x-oold-range`` references, + +- fall out for free when you generate from an explicit intermediate +representation (IR) instead. The schema graph is parsed once into ``ClassIR`` / +``FieldIR`` nodes; identity is resolved structurally by ``x-oold-uuid``; the +emitter then prints idiomatic pydantic once. There is **no** text +post-processing. + +The emitter targets the ``Ref[T]`` binding from ``ref_binding.py``, so the two +spikes compose: the recommended generator emits the recommended binding. + +Run it:: + + python -m oold.experimental.codegen_spike + +It prints the generated module, executes it, and self-checks the three +properties above. + +Scope: intentionally minimal (string/scalar types, the IRI form of +``x-oold-range``, single-file output). It is a feasibility probe, not the +Phase-2 generator. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +# Intermediate representation + + +@dataclass +class FieldIR: + name: str + py_type: str # e.g. "str", "Ref[Person]" + required: bool = False + default_repr: str | None = None # source text for the default, if any + + +@dataclass +class ClassIR: + name: str + uuid: str | None = None + x_oold_iri: str | None = None + bases: list[str] = field(default_factory=list) # resolved base class names + fields: list[FieldIR] = field(default_factory=list) + + +_JSON_TO_PY = { + "string": "str", + "integer": "int", + "number": "float", + "boolean": "bool", +} + + +def _read_range(prop: dict) -> object | None: + """Dual-read the range keyword (x-oold-range canonical, legacy 'range').""" + if "x-oold-range" in prop: + return prop["x-oold-range"] + return prop.get("range") + + +def _title_of_ref(ref: str) -> str: + """Reduce a $ref / range IRI to a schema title (last path segment, no ext).""" + tail = ref.rstrip("/").split("/")[-1] + for ext in (".schema.json", ".json"): + if tail.endswith(ext): + tail = tail[: -len(ext)] + return tail + + +# Front end: schema graph to IR + + +def build_ir(schemas: dict[str, dict]) -> list[ClassIR]: + """Turn a set of OO-LD schemas (keyed by title) into ordered ClassIR nodes. + + Identity is resolved by ``x-oold-uuid``: the first schema carrying a UUID + is canonical; any later schema with the same UUID is an *alias* and does + not produce its own class. References to an alias resolve to the canonical + class. This is the structural equivalent of the generator's UUID-dedup + regex passes. + """ + # 1. Resolve x-oold-uuid identity -> canonical class name per title. + uuid_to_canonical: dict[str, str] = {} + title_to_class: dict[str, str] = {} + for title, schema in schemas.items(): + uuid = schema.get("x-oold-uuid") or schema.get("uuid") + if uuid and uuid in uuid_to_canonical: + title_to_class[title] = uuid_to_canonical[uuid] # alias -> canonical + else: + cls_name = schema.get("title", title) + title_to_class[title] = cls_name + if uuid: + uuid_to_canonical[uuid] = cls_name + + def resolve(ref: str) -> str: + title = _title_of_ref(ref) + return title_to_class.get(title, title) + + # 2. Build one ClassIR per *canonical* title. + seen: set = set() + irs: list[ClassIR] = [] + for title, schema in schemas.items(): + cls_name = title_to_class[title] + if cls_name in seen: + continue # alias or duplicate - already represented + seen.add(cls_name) + + cir = ClassIR( + name=cls_name, + uuid=schema.get("x-oold-uuid") or schema.get("uuid"), + x_oold_iri=schema.get("x-oold-iri") or schema.get("iri"), + ) + + # allOf -> multiple inheritance (each $ref becomes a base class). + for entry in schema.get("allOf", []): + if "$ref" in entry: + cir.bases.append(resolve(entry["$ref"])) + + required = set(schema.get("required", [])) + for pname, prop in schema.get("properties", {}).items(): + cir.fields.append(_field_ir(pname, prop, pname in required, resolve)) + + irs.append(cir) + + return irs + + +def _field_ir(pname: str, prop: dict, required: bool, resolve) -> FieldIR: + rng = _read_range(prop) + if rng is not None: + # x-oold-range: typed reference. IRI form (str) and array-of-IRI form + # are handled; an inline-subschema form would recurse (out of scope). + if isinstance(rng, str): + target = resolve(rng) + elif isinstance(rng, list) and rng: + target = resolve(rng[0]) # union collapses to first for the spike + else: + target = "object" + py_type = f"Ref[{target}]" + else: + py_type = _JSON_TO_PY.get(prop.get("type", "string"), "str") + + default_repr = None + if "default" in prop: + default_repr = repr(prop["default"]) + elif not required: + default_repr = "None" + + return FieldIR( + name=pname, + py_type=py_type, + required=required, + default_repr=default_repr, + ) + + +# Back end: IR to pydantic source (single pass, no text post-processing) + + +def emit(irs: list[ClassIR]) -> str: + lines: list[str] = [ + '"""Generated by oold.experimental.codegen_spike - do not edit."""', + "from __future__ import annotations", + "", + "from typing import ClassVar, Optional", + "", + "from oold.model._ref import OoldModel, Ref", + "", + ] + for cir in irs: + bases = ", ".join(cir.bases) if cir.bases else "OoldModel" + lines.append(f"class {cir.name}({bases}):") + body_start = len(lines) + if cir.x_oold_iri: + lines.append(f" x_oold_iri: ClassVar[str] = {cir.x_oold_iri!r}") + for f in cir.fields: + ann = f.py_type + if f.default_repr is None: + lines.append(f" {f.name}: {ann}") + else: + ann = f"Optional[{ann}]" if f.default_repr == "None" else ann + lines.append(f" {f.name}: {ann} = {f.default_repr}") + if len(lines) == body_start: # no members emitted + lines.append(" pass") + lines.append("") + + # Rebuild models so self-referential Ref[...] forward refs resolve. This is + # generated *code*, driven by the IR (not a regex over output text). + for cir in irs: + lines.append(f"{cir.name}.model_rebuild()") + lines.append("") + return "\n".join(lines) + + +# Example schema graph + self-check + +EXAMPLE_SCHEMAS: dict[str, dict] = { + "Item": { + "title": "Item", + "x-oold-uuid": "11111111-1111-1111-1111-111111111111", + "x-oold-iri": "ex:Item", + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + }, + # Alias: same UUID as Item -> must NOT emit a second class; references to + # it resolve to Item. (Mirrors the generator's UUID-dedup.) + "Thing": { + "title": "Thing", + "x-oold-uuid": "11111111-1111-1111-1111-111111111111", + "type": "object", + "properties": {"id": {"type": "string"}}, + "required": ["id"], + }, + "Named": { + "title": "Named", + "x-oold-uuid": "22222222-2222-2222-2222-222222222222", + "type": "object", + "properties": {"label": {"type": "string"}}, + }, + # Multiple allOf -> class Person(Item, Named). best_friend is a typed + # self-referential x-oold-range (legacy bare 'range' also accepted). + "Person": { + "title": "Person", + "x-oold-uuid": "33333333-3333-3333-3333-333333333333", + "type": "object", + "allOf": [{"$ref": "Item.json"}, {"$ref": "Named.json"}], + "properties": { + "name": {"type": "string"}, + "best_friend": {"type": "string", "x-oold-range": "Person"}, + }, + }, + # References the alias 'Thing' -> resolves to base Item, proving reuse. + "Widget": { + "title": "Widget", + "x-oold-uuid": "44444444-4444-4444-4444-444444444444", + "type": "object", + "allOf": [{"$ref": "Thing.json"}], + "properties": {"watts": {"type": "number"}}, + }, +} + + +def main() -> None: + irs = build_ir(EXAMPLE_SCHEMAS) + source = emit(irs) + + print("=" * 70) + print("GENERATED MODULE") + print("=" * 70) + print(source) + + # Execute the generated module and validate the three target properties. + ns: dict[str, object] = {} + exec(compile(source, "", "exec"), ns) # noqa: S102 + + Item = ns["Item"] + Named = ns["Named"] + Person = ns["Person"] + Widget = ns["Widget"] + Ref = ns["Ref"] + + print("=" * 70) + print("SELF-CHECK") + print("=" * 70) + + # (1) allOf multiple inheritance + assert issubclass(Person, Item) and issubclass(Person, Named), Person.__mro__ + print("[ok] allOf -> multiple inheritance: Person(Item, Named)") + + # (2) x-oold-uuid reuse: 'Thing' aliased Item, so no Thing class and Widget + # inherits the canonical Item. + assert "Thing" not in ns, "alias 'Thing' must not emit its own class" + assert issubclass(Widget, Item), "Widget must reuse the canonical Item" + print("[ok] x-oold-uuid reuse: Thing collapsed into Item; Widget(Item)") + + # (3) typed x-oold-range reference, built from an IRI and lazily typed + p = Person(id="ex:alice", name="Alice", best_friend="ex:bob") + assert isinstance(p.best_friend, Ref), type(p.best_friend) + assert p.best_friend.iri == "ex:bob" + assert p.model_dump(exclude_none=True)["best_friend"] == "ex:bob" + print("[ok] x-oold-range -> Ref[Person]; best_friend serialises to IRI") + + # required propagates from the schema (id is required on Item) + try: + Person(name="no id") + except Exception: + print("[ok] required field 'id' enforced (inherited from Item)") + else: # pragma: no cover + raise AssertionError("expected validation error for missing required id") + + print("\nALL CHECKS PASSED") + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/src/oold/model/__init__.py b/src/oold/model/__init__.py index e43e362..99ae4ad 100644 --- a/src/oold/model/__init__.py +++ b/src/oold/model/__init__.py @@ -1,5 +1,6 @@ import json import logging +import os from typing import ( TYPE_CHECKING, Any, @@ -107,7 +108,49 @@ def __getattr__(self, name): _controller_types: dict[str, list] = {} -M = TypeVar("M", bound="LinkedBaseModel") +M = TypeVar("M", bound="_LinkedBaseModelLegacy") + + +def register_type(cls: type, iri: str | list[str] | None = None) -> None: + """Register a model class under the type IRI(s) it answers to. + + The public entry point for the type registry that resolution consults when + mapping a document's ``type`` back to a class. Classes register themselves + on creation, so this is only needed for classes built dynamically, aliased + under an extra IRI, or defined before their IRI is known. + + Without it callers reach for the private ``_types`` mapping and write into + it directly, which couples them to an internal and offers no validation. + + Parameters + ---------- + cls + The model class to register. + iri + The IRI(s) to register under. Defaults to ``cls.get_cls_iri()``. + """ + if iri is None: + iri = cls.get_cls_iri() if hasattr(cls, "get_cls_iri") else None + if iri is None: + raise ValueError(f"{cls.__name__} has no type IRI: pass iri= or define get_cls_iri()") + for value in iri if isinstance(iri, list) else [iri]: + if isinstance(value, str): + _types[value] = cls + + +def get_registered_type(iri: str) -> type | None: + """The model class registered for a type IRI, or ``None``.""" + return _types.get(iri) + + +def registered_types() -> dict: + """The live type registry. + + The same mapping resolution uses; mutating it affects resolution. Prefer + :func:`register_type` over writing to it directly. + """ + return _types + _logger = logging.getLogger(__name__) @@ -152,7 +195,7 @@ def _inherited_cls_iris(cls) -> frozenset: # pydantic v2 -class LinkedBaseModelMetaClass(pydantic.main._model_construction.ModelMetaclass): +class _LinkedBaseModelMetaClassLegacy(pydantic.main._model_construction.ModelMetaclass): _constructing: bool = False """Guards against __getattribute__ intercepting field access during class construction. Pydantic checks ``getattr(base, field_name, None)`` in its @@ -161,11 +204,11 @@ class LinkedBaseModelMetaClass(pydantic.main._model_construction.ModelMetaclass) of the default None, causing false-positive field-name collision errors.""" def __new__(mcs, name, bases, namespace, **kwargs): - LinkedBaseModelMetaClass._constructing = True + _LinkedBaseModelMetaClassLegacy._constructing = True try: cls = super().__new__(mcs, name, bases, namespace, **kwargs) finally: - LinkedBaseModelMetaClass._constructing = False + _LinkedBaseModelMetaClassLegacy._constructing = False # Register type IRI mapping. Controllers go to _controller_types # (they extend data models but should not replace them in @@ -393,8 +436,14 @@ def __getattribute__(self, name): # class LinkedBaseModel(_LinkedBaseModel): -class LinkedBaseModel(BaseModel, GenericLinkedBaseModel, metaclass=LinkedBaseModelMetaClass): - """LinkedBaseModel for pydantic v2""" +class _LinkedBaseModelLegacy(BaseModel, GenericLinkedBaseModel, metaclass=_LinkedBaseModelMetaClassLegacy): + """The per-attribute-interception binding. + + Exported as ``LinkedBaseModel`` unless ``OOLD_DESCRIPTOR_BINDING=0`` selects + it, which is decided at the bottom of this module. It carries its own name + so that the exported one has a single declaration a type checker can + resolve - pyright keeps the first of two and would ignore the swap. + """ __iris__: dict[str, str | list[str]] | None = {} @@ -729,10 +778,10 @@ def _recursive_object_to_iri(d: dict, model_obj): for item, model_item in zip(value, model_value, strict=False): if isinstance(item, dict) and hasattr(model_item, "__iris__"): model_item._object_to_iri(item) - LinkedBaseModel._recursive_object_to_iri(item, model_item) + _LinkedBaseModelLegacy._recursive_object_to_iri(item, model_item) elif isinstance(value, dict) and hasattr(model_value, "__iris__"): model_value._object_to_iri(value) - LinkedBaseModel._recursive_object_to_iri(value, model_value) + _LinkedBaseModelLegacy._recursive_object_to_iri(value, model_value) def get_iri_ref(self, field_name: str): """Return the stored IRI reference string(s) for a field without @@ -778,7 +827,7 @@ def get_raw(self, field_name: str): @staticmethod def _resolve(iris): resolver = get_resolver(GetResolverParam(iri=iris[0])).resolver - node_dict = resolver.resolve(ResolveParam(iris=iris, model_cls=LinkedBaseModel)).nodes + node_dict = resolver.resolve(ResolveParam(iris=iris, model_cls=_LinkedBaseModelLegacy)).nodes return node_dict def _store(self): @@ -988,9 +1037,9 @@ def to_jsonld(self) -> dict: return export_jsonld(self, BaseModel) @classmethod - def from_jsonld(cls, jsonld: dict) -> "LinkedBaseModel": + def from_jsonld(cls, jsonld: dict) -> "_LinkedBaseModelLegacy": """Constructs a model instance from a JSON-LD representation.""" - return import_jsonld(BaseModel, LinkedBaseModel, cls, jsonld, _types) + return import_jsonld(BaseModel, _LinkedBaseModelLegacy, cls, jsonld, _types) def to_json(self, exclude_defaults: bool = False) -> dict: """Return the JSON representation of the object. @@ -1020,9 +1069,9 @@ def to_json(self, exclude_defaults: bool = False) -> dict: return result @classmethod - def from_json(cls, data: dict) -> "LinkedBaseModel": + def from_json(cls, data: dict) -> "_LinkedBaseModelLegacy": """Constructs a model instance from a JSON representation.""" - return import_json(BaseModel, LinkedBaseModel, cls, data, _types) + return import_json(BaseModel, _LinkedBaseModelLegacy, cls, data, _types) # @classmethod # def model_json_schema( @@ -1075,12 +1124,23 @@ def _get_data_model_cls(self): """ def _is_data_model(cls): + # A data model is recognised by carrying fields, not only by not + # being on a name list: the descriptor binding mixes in + # LinkedApiMixin, which answers to_json/from_json but declares no + # fields, and a name-only test picked it as the data model - so + # to_json() intersected against an empty field set and returned + # nothing but the type. + fields = getattr(cls, "model_fields", None) + if fields is None: # pydantic v1 classes + fields = getattr(cls, "__fields__", None) return ( cls is not type(self) + and bool(fields) and cls.__name__ not in ( "LinkedBaseModel", "_LinkedBaseModel", + "_LinkedBaseModelLegacy", "BaseController", "GenericLinkedBaseModel", "BaseModel", @@ -1182,3 +1242,67 @@ def to_jsonld(self): ): del data[key] return data + + +# --------------------------------------------------------------------------- +# The descriptor binding +# --------------------------------------------------------------------------- +# The descriptor binding (see docs/design/graph-object-binding.md) replaces the +# per-attribute interception above with one descriptor per link field, and is +# what LinkedBaseModel means. It was made the default once before on the +# strength of three downstream suites passing; a review then found seven +# behaviours it did not reproduce, none of which those suites reached. Each is +# now covered by a test in tests/test_compat_parity.py that fails without its +# fix. The legacy binding remains one environment variable away: +# +# OOLD_DESCRIPTOR_BINDING=0 +# +# Two names move with the base class, because downstream imports them and relies +# on their identity (see docs/design/downstream-migration.md): +# +# * LinkedBaseModelMetaClass - subclassed downstream, so a derived metaclass +# must remain a subclass of whatever LinkedBaseModel actually uses; +# * _types - written to downstream, so the binding must share the very same +# mapping rather than keep its own. +# +# Which of the two a name refers to is decided at import time, and a type +# checker cannot follow that. Left to infer, it keeps the legacy types for every +# consumer of this module - so `Model[Model.field == x]` reads as +# `Model | LinkedBaseModelList[Model]`, and iterating it yields pydantic's +# `tuple[str, Any]` instead of the model. The TYPE_CHECKING branch below states +# the default statically, so the typed subscript and link annotations reach code +# importing from `oold.model` rather than only from the private module. +if TYPE_CHECKING: + from oold.model._descriptor import LinkedBaseModel as LinkedBaseModel + from oold.model._descriptor import ( + LinkedBaseModelMetaClass as LinkedBaseModelMetaClass, + ) +elif os.environ.get("OOLD_DESCRIPTOR_BINDING", "1") != "0": + from oold.model import _descriptor as _descriptor_module + + _descriptor_module.use_type_registry(_types) + LinkedBaseModel = _descriptor_module.LinkedBaseModel + LinkedBaseModelMetaClass = _descriptor_module.LinkedBaseModelMetaClass +else: # pragma: no cover + _logger.info("oold: legacy binding selected (OOLD_DESCRIPTOR_BINDING=0)") + LinkedBaseModel = _LinkedBaseModelLegacy + LinkedBaseModelMetaClass = _LinkedBaseModelMetaClassLegacy + +# The link notations are part of the public surface either way: importing them +# from a private module is not something an example should have to do. They are +# only *effective* with the descriptor binding, which is what LINK_NOTATIONS_ACTIVE +# reports - the shipped binding above reads `range` and ignores a Link[...] +# annotation. +from oold.model._descriptor import Link as Link # noqa: E402 +from oold.model._descriptor import LinkList as LinkList # noqa: E402 +from oold.model._descriptor import LinkNotResolved as LinkNotResolved # noqa: E402 +from oold.model._descriptor import LinkResultList as LinkResultList # noqa: E402 +from oold.model._descriptor import OoldExtra as OoldExtra # noqa: E402 +from oold.model._descriptor import OoldField as OoldField # noqa: E402 + +LINK_NOTATIONS_ACTIVE = LinkedBaseModel is not _LinkedBaseModelLegacy +"""Whether ``Link[T]`` / ``LinkList[T]`` annotations are honoured. + +True unless ``OOLD_DESCRIPTOR_BINDING=0``: the legacy binding recognises links +only through the ``range`` keyword. +""" diff --git a/src/oold/model/_compat.py b/src/oold/model/_compat.py new file mode 100644 index 0000000..fb38247 --- /dev/null +++ b/src/oold/model/_compat.py @@ -0,0 +1,311 @@ +"""Public-API parity layer for the descriptor binding. + +Downstream code (the generated ``opensemantic.*`` packages and the applications +built on them) inherits its API from ``oold.model.LinkedBaseModel`` via +``OswBaseModel``. Replacing the binding therefore has to keep that surface +working. A scan of those code bases found these members in active use: + +=========================== ===== ================================= +member sites note +=========================== ===== ================================= +``get_iri_ref`` 24 hand-written application code +``__iris__`` 15 read **and written** by callers +``get_cls_iri`` 42 inherited, unchanged +``to_json`` / ``from_json`` 9 each +``to_jsonld`` / ``from_jsonld`` 4 / 1 +``cast`` / ``cast_none_to_default`` 3 / 2 +``get_raw`` 2 +=========================== ===== ================================= + +``LinkedBaseModelList`` and ``store_jsonld`` had no downstream hits. + +The mixin below re-implements that surface on top of the descriptor storage +(``_links``), reusing :mod:`oold.static` for the RDF work so behaviour matches +the shipped model rather than being re-derived. + +``__iris__`` is a read/write property: the shipped model exposes a plain dict +and callers assign to it directly, e.g. in ``opensemantic.base``:: + + self.__iris__ = {"characteristic": characteristic_class.get_cls_iri()} + +so a read-only shim would silently drop such assignments. +""" + +from __future__ import annotations + +import json +from typing import Any + +from pydantic import BaseModel + +from oold.static import ( + GenericLinkedBaseModel, + export_jsonld, + import_json, + import_jsonld, +) + + +def _raw_of(stored: Any) -> Any: + """The nested form of references that carry no IRI.""" + + def one(ref: Any) -> Any: + obj = getattr(ref, "_obj", None) if ref is not None else None + if obj is None: + return None + return obj._raw_dict() if hasattr(obj, "_raw_dict") else obj + + if isinstance(stored, list): + out = [one(r) for r in stored] + return out or None + return one(stored) + + +def _is_model(value: Any) -> bool: + return hasattr(value, "_dump") or hasattr(value, "model_dump") or hasattr(value, "dict") + + +def _plain_dump(value: Any) -> Any: + """Serialise a nested model, whichever kind it is. + + Testing for the ``_dump`` hook alone misses **plain** pydantic models - the + hook only exists on LinkedApiMixin - so a nested BaseModel came back as the + object rather than a dict, and cast() then handed it to the target + constructor. + """ + for attr in ("_dump", "model_dump", "dict"): + fn = getattr(value, attr, None) + if callable(fn): + return fn() + return value + + +def _drop_iri(stored: Any) -> None: + """Forget the IRI of a stored reference, keeping any object it holds.""" + refs = stored if isinstance(stored, list) else [stored] + for ref in refs: + if ref is not None: + ref.iri = None + + +class LinkedApiMixin(GenericLinkedBaseModel): + """Re-implements the shipped ``LinkedBaseModel`` API over ``_links``. + + Shared by both pydantic versions. Everything version-specific goes through + the three hooks below, so the members that differ only in ``model_fields`` + vs ``__fields__`` - which was nine of them, at 0.97 to 1.00 similarity - + live here once rather than being forked per version. + """ + + @classmethod + def _fields(cls) -> dict: + """The declared fields: ``model_fields`` in v2, ``__fields__`` in v1.""" + return cls.model_fields + + def _dump(self, **kwargs: Any) -> dict: + """A plain dict of the model: ``model_dump`` in v2, ``dict`` in v1.""" + return self.model_dump(**kwargs) + + # -- reference inspection, no resolution -------------------------------- + + @property + def __iris__(self) -> dict[str, Any]: + """The stored IRI reference(s) per link field. + + Mirrors the shipped side-dict. Writable: assigning a mapping replaces + the stored references, which is what external code relies on. + """ + out: dict[str, Any] = {} + for name, descr in type(self).__link_fields__.items(): + iris = descr.iris(self) + if iris: + out[name] = iris + out.update(self._extra_iris) + return out + + @__iris__.setter + def __iris__(self, value: dict[str, Any]) -> None: + """Replace the stored *references*, and only those. + + The shipped side-dict is a plain attribute, so assigning to it drops + whatever was there - merging would silently keep links the caller meant + to clear, and ``= {}`` would do nothing at all. + + Two things it must not do. Destroy a value: a ``Ref`` carries both an + IRI and the object once it has one, so clearing the slot outright would + lose a resolved or inline object the side-dict never held - only the IRI + goes. And overwrite a field: a key that is not a link field is remembered + as a reference rather than written over the model field of that name. + """ + link_fields = type(self).__link_fields__ + value = value or {} + for name in link_fields: + if name in value: + continue + _drop_iri(self._links.get(name)) + self.__dict__.pop(name, None) + for name, iris in value.items(): + descr = link_fields.get(name) + if descr is None: + self._extra_iris[name] = iris + continue + descr.set_value(self, iris) + + @classmethod + def get_cls_iri(cls) -> Any: + """The class IRI(s), from the schema annotation and the type default. + + ``GenericLinkedBaseModel`` only declares this abstract, so without an + implementation it silently returns ``None`` - which would break the + downstream callers and the type registry alike. + """ + schema = getattr(cls, "model_config", {}).get("json_schema_extra") or {} + if callable(schema): + schema = {} + out: list[str] = [] + for key in ("$id", "x-oold-iri", "iri"): + if key in schema: + out.append(schema[key]) + break + type_field = cls._fields().get(cls.get_type_field()) + if type_field is not None: + # A list default is a type *array*: the class answers to every IRI + # in it, so flatten. Appending the list as one element left a + # non-string in the result, which _register_class skips - so a class + # with a type array registered under its $id only and was + # unreachable by type. v1 already flattened, as does _iri_set. + default = type_field.default + for value in default if isinstance(default, list) else [default]: + if value is not None and value not in out: + out.append(value) + if not out: + return None + return out[0] if len(out) == 1 else out + + def get_iri(self) -> str | None: + """The instance IRI. Overridden by models that derive it differently.""" + return getattr(self, "id", None) + + def link_iris(self, name: str) -> Any: + """The stored reference(s) for one link, without resolving.""" + return type(self).__link_fields__[name].iris(self) + + def get_iri_ref(self, field_name: str) -> Any: + """IRI reference(s) for a field, or ``None``, without resolving.""" + iris = self.__iris__.get(field_name) + if iris is None: + return None + if isinstance(iris, list): + return iris if iris else None + return iris + + def get_raw(self, field_name: str) -> Any: + """The stored value without triggering resolution.""" + descr = type(self).__link_fields__.get(field_name) + if descr is None: + return self.__dict__.get(field_name) + stored = self._links.get(field_name) + if isinstance(stored, list): + # filter on the resolved object, not on the Ref: an unresolved + # reference has a Ref but no object, and answering [None] would read + # as "there is one, and it is nothing" + objs = [r._obj for r in stored if r is not None and r._obj is not None] + return objs or None + return stored._obj if stored is not None else None + + # -- serialisation ------------------------------------------------------ + + def _raw_dict(self) -> dict[str, Any]: + """Serialise without resolving; links become IRI strings. + + Mirrors the shipped ``_raw_dict``: **every** declared field appears, with + ``None`` where unset. ``cast()`` is built on this, so omitting empty + fields would silently drop them from the target. + """ + links = type(self).__link_fields__ + d: dict[str, Any] = {} + for name in type(self)._fields(): + if name in links: + iri = self.get_iri_ref(name) + if iri is None: + # No IRI to reference. An inline object that has not been + # given one still has to appear, or cast() drops it - the + # shipped _raw_dict keeps it, nested. + iri = _raw_of(self._links.get(name)) + d[name] = iri + continue + value = self.__dict__.get(name) + if isinstance(value, list): + d[name] = [v._raw_dict() if hasattr(v, "_raw_dict") else _plain_dump(v) for v in value] + elif hasattr(value, "_raw_dict"): + d[name] = value._raw_dict() + elif _is_model(value): + d[name] = _plain_dump(value) + else: + d[name] = value + return d + + def to_json(self, exclude_defaults: bool = False) -> dict[str, Any]: + result = json.loads(self.model_dump_json(exclude_none=True, exclude_defaults=exclude_defaults)) + for name in type(self).__link_fields__: + iri = self.get_iri_ref(name) + if iri is not None and not result.get(name): + result[name] = iri + return result + + @classmethod + def _root_cls(cls) -> type: + from oold.model._descriptor import LinkedBaseModel + + return LinkedBaseModel + + @classmethod + def from_json(cls, data: dict[str, Any]) -> Any: + from oold.model._descriptor import _TYPE_REGISTRY + + # root must be the binding base: import_json only falls back to the + # given class when the two differ, which is how a payload without a + # type IRI still constructs. + return import_json(BaseModel, cls._root_cls(), cls, data, _TYPE_REGISTRY) + + def to_jsonld(self) -> dict[str, Any]: + return export_jsonld(self, BaseModel) + + @classmethod + def from_jsonld(cls, jsonld: dict[str, Any]) -> Any: + from oold.model._descriptor import _TYPE_REGISTRY + + return import_jsonld(BaseModel, cls._root_cls(), cls, jsonld, _TYPE_REGISTRY) + + def store_jsonld(self) -> None: + from oold.backend.interface import GetBackendParam, StoreParam, get_backend + + backend = get_backend(GetBackendParam(iri=self.get_iri())).backend + backend.store(StoreParam(nodes={self.get_iri(): self})) + + # -- conversion --------------------------------------------------------- + + def cast( + self, + cls: type, + none_to_default: bool = False, + remove_extra: bool = False, + silent: bool = True, + **kwargs: Any, + ) -> Any: + data = {**self._raw_dict(), **kwargs} + if none_to_default: + data = { + k: v + for k, v in data.items() + if v is not None and not (isinstance(v, list) and not [x for x in v if x is not None]) + } + if remove_extra: + target = set(cls._fields() if hasattr(cls, "_fields") else getattr(cls, "model_fields", {})) + if target: + data = {k: v for k, v in data.items() if k in target} + data.pop("type", None) + return cls(**data) + + def cast_none_to_default(self, cls: type, **kwargs: Any) -> Any: + return self.cast(cls, none_to_default=True, **kwargs) diff --git a/src/oold/model/_descriptor.py b/src/oold/model/_descriptor.py new file mode 100644 index 0000000..3b60816 --- /dev/null +++ b/src/oold/model/_descriptor.py @@ -0,0 +1,1502 @@ +"""The descriptor graph-object binding, with no declaration syntax change. + +Models are declared exactly as they are today - standard annotations, including +plain ``List[...]`` for to-many links: + + class Person(LinkedBaseModel): + id: str + name: Optional[str] = None + knows: Optional[List["Person"]] = Field( + None, json_schema_extra={"x-oold-range": "Person"} + ) + +so the code ``datamodel-code-generator`` already emits keeps working untouched. +``Link[T]`` / ``LinkList[T]`` are available on top for declarations that should +also be typed in both directions - see ``docs/design/graph-object-binding.md``. + +After pydantic finishes building the class, ``__pydantic_init_subclass__`` scans +``model_fields`` for a ``x-oold-range`` (or legacy ``range``) annotation and +installs a descriptor for each such field. Because attribute lookup consults the +type before the instance ``__dict__``, the descriptor handles link reads while +every other field keeps native pydantic access: the "is this a range field?" +test is performed by the interpreter's C-level attribute lookup rather than a +Python ``__getattribute__``, so plain fields cost nothing. + +The descriptor is deliberately **non-data** - it defines ``__get__`` but no +runtime ``__set__`` - and caches the resolved value in the instance ``__dict__``, +which then shadows it. Warm link reads are therefore a plain dict lookup that +never re-enters Python (the ``functools.cached_property`` pattern, worth 32x); +writes are intercepted by ``__setattr__`` instead, which drops the cache entry. + +Semantics match the shipped binding: reading a link returns the **real** resolved +object (``isinstance`` holds), resolution is lazy, and references serialise back +to IRIs. Resolution is additionally **batched** - a list resolves in one backend +call. + +Selected with ``OOLD_DESCRIPTOR_BINDING=1``; ``OOLD_LINKS=0`` turns link +behaviour off entirely and leaves plain pydantic. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +import types +from collections import defaultdict +from collections.abc import Iterable, Mapping +from typing import ( + TYPE_CHECKING, + Annotated, + Any, + ClassVar, + Generic, + SupportsIndex, + TypeVar, + Union, + get_args, + get_origin, + overload, +) + +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, SerializationInfo, model_serializer +from pydantic._internal._model_construction import ModelMetaclass +from pydantic_core import PydanticUndefined + +from oold.backend import interface +from oold.backend.interface import ( + Condition, + GetResolverParam, + QueryParam, + ResolveParam, + apply_operator, + get_resolver, +) +from oold.model._compat import LinkedApiMixin +from oold.model._ref import Ref, _construct + +T = TypeVar("T") +_M = TypeVar("_M") + + +class _Constructing: + """Whether a model class is being built, for either pydantic version. + + One flag, not one per version: the descriptor's ``__get__`` consults it, and + that descriptor is shared, so two flags would mean v1 class construction set + one while the guard read the other. Pydantic probes the bases for + same-named attributes while building a class, and an installed descriptor is + exactly such an attribute - without this the probe finds it and rejects the + field (v2 warns and mis-assigns, v1 raises NameError). + + A **counter**, not a boolean: class bodies nest. A model defined while + another is being built - a lazy import, ``create_model`` from a metaclass + hook, a forward reference resolved mid-build - would otherwise clear the + flag on its way out and leave the enclosing build unguarded. + """ + + depth: int = 0 + + @classmethod + def enter(cls) -> None: + cls.depth += 1 + + @classmethod + def leave(cls) -> None: + cls.depth = max(0, cls.depth - 1) + + @classmethod + def is_active(cls) -> bool: + return cls.depth > 0 + + +class LinkNotResolved(LookupError): + """A mandatory link did not yield an object. + + Raised only for links declared ``Link[T]`` rather than ``Link[T | None]``: + the declaration promises a ``T``, so handing back a ``None`` the type denies + would be the real error. Declare the ``None`` arm where absence is data. + """ + + +def links_enabled() -> bool: + """Whether OO-LD link behaviour is active. + + ``OOLD_LINKS=0`` turns it off: no descriptors are installed, nothing is + routed out of the payload, and serialisation is pydantic's own. Useful to + check whether a problem is OO-LD's or the model's, and to run a code base + as plain pydantic without editing it. + """ + return os.environ.get("OOLD_LINKS", "1") != "0" + + +def _neutralise_link_defaults(namespace: dict) -> dict[str, Any]: + """Make link fields optional and defaultless at the pydantic level. + + Link values are routed around pydantic - the descriptor holds them - so the + field is always absent from the payload pydantic validates, and a default + left in place would be evaluated on every construction. Generated models + spell a default IRI as ``T.model_validate("")``, which resolves through + the backend, so leaving it would also turn every construction into a + synchronous fetch. + + The IRI itself is not dead weight, though - dropping it outright lost the + declared default. It is recorded in ``__link_defaults__`` and handed to the + descriptor on construction, so the link resolves lazily, like any other. + + This runs on the class namespace rather than on ``model_fields``, because by + the time ``__pydantic_init_subclass__`` sees the fields the core schema - + defaults included - has already been built. + """ + import copy as _copy + + defaults: dict[str, Any] = {} + + def _is_link_field_info(info: Any) -> bool: + extra = getattr(info, "json_schema_extra", None) + if not isinstance(extra, dict): + return False + return bool(extra.get("x-oold-range") or extra.get("range") or extra.get("x-oold-link")) + + def _record_default(field_name: str, info: Any) -> None: + iris = _default_iris(info) + if iris is not None: + defaults[field_name] = iris + + def _neutralised(info: Any) -> Any: + # Copy first: a FieldInfo can be shared between models (a module-level + # SHARED = Field(...) assigned to several classes), and mutating it in + # place stripped that default process-wide, including from plain + # BaseModels that have nothing to do with links. + info = _copy.copy(info) + info.default = None + info.default_factory = None + # FieldInfo.from_annotated_attribute rebuilds the field from + # _attributes_set, so clearing the live attributes alone has no effect + attributes_set = getattr(info, "_attributes_set", None) + if isinstance(attributes_set, dict): + attributes_set = dict(attributes_set) + attributes_set.pop("default_factory", None) + attributes_set["default"] = None + info._attributes_set = attributes_set + return info + + annotations = _namespace_annotations(namespace) + for field_name, annotation in annotations.items(): + info = namespace.get(field_name) + if _is_link_field_info(info): + _record_default(field_name, info) + namespace[field_name] = _neutralised(info) + continue + if field_name not in namespace and _is_link_annotation(annotation): + # `manager: Link[Org]` with nothing assigned. Read as Python reads + # it - no default means required - but a link cannot be required as + # a pydantic field, because its value never reaches validation. + # Left alone, every construction failed with a misleading + # "Field required" about a value that had in fact been supplied. + namespace[field_name] = OoldField(required=True) + continue + # A Field() living in Annotated metadata rather than as the assigned + # value was never seen here, so its default survived and was evaluated + # on every construction - the very failure this function exists to stop. + if get_origin(annotation) is not Annotated: + continue + args = get_args(annotation) + for meta in args[1:]: + if _is_link_field_info(meta): + _record_default(field_name, meta) + rebuilt = [_neutralised(m) if _is_link_field_info(m) else m for m in args[1:]] + if rebuilt != list(args[1:]): + # materialise the dict before writing: under PEP 649 the namespace + # has only __annotate__, and an explicit __annotations__ takes + # precedence over it + if "__annotations__" not in namespace: + namespace["__annotations__"] = dict(annotations) + namespace["__annotations__"][field_name] = Annotated[(args[0], *rebuilt)] + if field_name not in namespace: + # Annotated-only declarations are required at the pydantic + # level; the value is routed to the descriptor, so give it the + # same absent default the assigned form gets. + namespace[field_name] = None + return defaults + + +def _namespace_annotations(namespace: dict) -> dict: + """The annotations of a class body being built, on any Python version. + + Python 3.14 defers them (PEP 649): the namespace carries an ``__annotate__`` + function instead of an ``__annotations__`` dict, so reading the dict found + nothing and every link default was left in place - which on 3.14 meant the + generated ``T.model_validate("")`` default was evaluated on every + construction, exactly what stripping it exists to prevent. + + The dict is still there when the model's module uses ``from __future__ + import annotations``, so it wins when non-empty. Otherwise the annotate + function is reached through ``annotationlib``, whose accessor hides the + namespace key - which was spelled ``__annotate__`` early in 3.14 and + ``__annotate_func__`` later, so reading it directly works on one and not the + other. + + FORWARDREF format, because a link annotation names its target by string more + often than not and VALUE would raise on the unresolved name. + """ + annotations = namespace.get("__annotations__") + if annotations: + return annotations + if sys.version_info < (3, 14): # before PEP 649 the dict is the only source + return annotations or {} + import annotationlib + + getter = getattr(annotationlib, "get_annotate_from_class_namespace", None) or getattr( + annotationlib, "get_annotate_function", None + ) + annotate = getter(namespace) if getter is not None else None + if annotate is None: + annotate = namespace.get("__annotate_func__") or namespace.get("__annotate__") + if annotate is None: + return annotations or {} + try: + return annotationlib.call_annotate_function(annotate, format=annotationlib.Format.FORWARDREF) or {} + except Exception: + return annotations or {} + + +def _default_iris(info: Any) -> Any: + """The IRI(s) a link field declares as its default, or ``None``. + + Two shapes reach here. A plain ``default="ex:b"`` is the IRI already. + Generated code instead emits + ``default_factory=lambda: Bar.model_validate("ex:b")``, where the IRI is a + constant in the lambda body - calling it would resolve through the backend, + which is exactly the eager fetch the binding exists to avoid, so the + constant is read instead. Anything else contributes no default. + """ + default = getattr(info, "default", None) + if isinstance(default, str) and default: + return default + if isinstance(default, list) and default and all(isinstance(v, str) and v for v in default): + return list(default) + factory = getattr(info, "default_factory", None) + code = getattr(factory, "__code__", None) + if code is None or code.co_argcount: + return None + iris = [c for c in code.co_consts if isinstance(c, str) and c] + if not iris: + return None + return iris[0] if len(iris) == 1 else iris + + +class OoldExtraModel(BaseModel): + """Validated model behind :class:`OoldExtra` (constraints live here).""" + + model_config = ConfigDict(populate_by_name=True, extra="allow") + + range: str = Field(alias="x-oold-range", min_length=1) + required_iri: bool | None = Field(None, alias="x-oold-required-iri") + + +class OoldExtra(dict[str, Any]): + """Typed, pydantic-validated replacement for a raw ``json_schema_extra`` dict. + + Must subclass ``dict``: pydantic merges ``json_schema_extra`` into the JSON + schema only via ``isinstance(json_schema_extra, dict)``, so a plain + ``BaseModel`` would be silently dropped from the schema. + + Validation is delegated to :class:`OoldExtraModel`, so real + ``ValidationError`` s are raised at declaration time, while typed properties + give checked read access instead of stringly-typed ``extra["x-oold-range"]``. + """ + + def __init__( + self, + *, + range: str, + required_iri: bool | None = None, + **vendor: Any, + ) -> None: + data: dict[str, Any] = {"x-oold-range": range} + if required_iri is not None: + data["x-oold-required-iri"] = required_iri + data.update(vendor) + # model_validate (not kwargs) keeps aliased names out of the call + # signature, which otherwise confuses type checkers. + model = OoldExtraModel.model_validate(data) + object.__setattr__(self, "_model", model) + super().__init__(model.model_dump(by_alias=True, exclude_none=True)) + + @property + def model(self) -> OoldExtraModel: + return self._model # type: ignore[attr-defined] + + @property + def range(self) -> str: + return self.model.range + + @property + def required_iri(self) -> bool | None: + return self.model.required_iri + + +def OoldField( + *, + range: str | None = None, + link: bool | None = None, + required: bool | None = None, + required_iri: bool | None = None, + **kwargs: Any, +) -> Any: + """``Field`` wrapper marking a property as a link. + + Keyword-only, and every argument is optional:: + + OoldField(range=None, link=None, required=None, **field_kwargs) + + ``required`` + The link must be supplied when the model is constructed; omitting it + raises ``ValueError``. Written to the schema as ``x-oold-required-iri`` + and into the standard ``required`` array. + + This is the knob for requiredness, **not** the annotation. The two are + different questions - "must the caller supply it?" and "what do I get + when I read it?" - and a self-referential link needs them to differ: + ``father: Link["Person"]`` reads as a ``Person`` so an ancestry walk + needs no guard per hop, while no real dataset can require every person + to name a father. + ``range`` + Target schema IRI, written as ``x-oold-range``. **Do not pass it**: it + is derived from the annotation, which already names the target, and + stating it twice lets the two disagree. + ``link`` + Marks the property a link where the annotation cannot, as in a union + arm: ``str | Location | None = OoldField(link=True)``. Redundant with + ``Link[T]`` / ``LinkList[T]``. + ``required_iri`` + Deprecated spelling of ``required``, kept because generated packages + pass it. The emitted keyword is unchanged. + ``**field_kwargs`` + Passed to ``pydantic.Field`` (``alias``, ``description``, + ``default_factory`` ...). ``default=None`` is supplied unless a + ``default_factory`` is given: link values are routed out of the payload + before pydantic validates, so a link field cannot be required *as a + pydantic field* - which is what ``required`` exists to express. + + The recommended declaration pairs it with :class:`Link` / :class:`LinkList`, + which are what give a type checker both the read and the write type:: + + father: Link["Person"] = OoldField() # chains guard-free + manager: Link[Organization] = OoldField(required=True) + advisor: Link["Organization | None"] = OoldField() # may read as None + """ + if required is None: + required = required_iri + elif required_iri is not None and bool(required_iri) != bool(required): + raise ValueError("OoldField: required and required_iri disagree; pass only required=") + if range is not None: + extra: dict[str, Any] = dict(OoldExtra(range=range, required_iri=required)) + else: + extra = {"x-oold-link": True if link is None else bool(link)} + if required is not None: + extra["x-oold-required-iri"] = required + # Link values are routed out of the payload before pydantic validates, so a + # link field must not be required at the pydantic level. This also makes the + # bare OoldField() form work with no arguments at all - but only when the + # caller has not supplied a factory, since pydantic rejects both at once. + if "default_factory" not in kwargs: + kwargs.setdefault("default", None) + return Field(**kwargs, json_schema_extra=extra) + + +def _has_default(value: Any) -> bool: + """Whether a field default is a real value rather than a placeholder.""" + return value is not None and value is not ... and value is not PydanticUndefined + + +class FieldProxy: + """Class-level field handle enabling ``Person.name == "John"``. + + Carries the field's default as well as its name, because the legacy proxy + did: downstream writes ``if Model.field:`` and ``Model.type.startswith(...)`` + against class attributes, which resolve to a proxy rather than to the + default. Without ``__bool__`` every such test is unconditionally true, and + without ``__getattr__`` every such call raises. + """ + + __slots__ = ("default", "name") + + def __init__(self, name: str, default: Any = None): + self.name = name + self.default = default + + def __bool__(self) -> bool: + return bool(self.default) if _has_default(self.default) else False + + def __getattr__(self, item: str) -> Any: + default = object.__getattribute__(self, "default") + if _has_default(default): + return getattr(default, item) + raise AttributeError(f"{type(self).__name__!r} object has no attribute {item!r}") + + def __eq__(self, other: Any) -> Any: # type: ignore[override] + return Condition(field=self.name, operator="eq", value=other) + + def __ne__(self, other: Any) -> Any: # type: ignore[override] + return Condition(field=self.name, operator="ne", value=other) + + def __lt__(self, other: Any) -> Any: + return Condition(field=self.name, operator="lt", value=other) + + def __le__(self, other: Any) -> Any: + return Condition(field=self.name, operator="le", value=other) + + def __gt__(self, other: Any) -> Any: + return Condition(field=self.name, operator="gt", value=other) + + def __ge__(self, other: Any) -> Any: + return Condition(field=self.name, operator="ge", value=other) + + def __hash__(self) -> int: + return id(self) + + +class LinkedBaseModelMetaClass(ModelMetaclass): + """Metaclass providing the query DSL without touching attribute reads. + + Uses ``__getattr__`` (a fallback, invoked only when normal lookup *fails*) + rather than ``__getattribute__`` (invoked on *every* access). Pydantic v2 + removes field names from the class namespace, so ``Person.name`` fails + naturally and lands here at no cost to any other attribute access. + """ + + _constructing = False + """Deprecated alias. The state lives on :class:`_Constructing`; this stays a + plain ``False`` so downstream ``if LinkedBaseModelMetaClass._constructing:`` + keeps meaning what it did, rather than becoming permanently true.""" + """Set while a class is being built. + + Pydantic probes ``getattr(base, field_name, None)`` during class + construction to detect shadowed attributes and inherited defaults. Since + field names are exactly what ``__getattr__`` answers with a FieldProxy, an + unguarded proxy is mistaken for an inherited default and ends up as the + field's value. Same reason the shipped metaclass carries this flag. + """ + + def __new__(mcs, name, bases, namespace, **kwargs): + defaults: dict[str, Any] = {} + if links_enabled(): + defaults = _neutralise_link_defaults(namespace) + _Constructing.enter() + try: + cls = super().__new__(mcs, name, bases, namespace, **kwargs) + finally: + _Constructing.leave() + if defaults: + # a subclass may add defaults without restating the inherited ones + cls.__link_defaults__ = {**getattr(cls, "__link_defaults__", {}), **defaults} + return cls + + def __getattr__(cls, name: str) -> Any: + # Never call getattr(cls, ...) here: cls.model_fields is a property + # that itself calls getattr, which would recurse until the stack blows. + if _Constructing.is_active(): + raise AttributeError(name) + if name.startswith("_"): + raise AttributeError(name) + for klass in cls.__mro__: + fields = klass.__dict__.get("__pydantic_fields__") + if fields and name in fields: + return FieldProxy(name, getattr(fields[name], "default", None)) + raise AttributeError(name) + + @overload + def __getitem__(cls: type[_M], item: str) -> _M | None: ... + + @overload + def __getitem__(cls: type[_M], item: Condition | bool) -> LinkResultList[_M] | None: ... + + @overload + def __getitem__(cls: type[_M], item: list[str]) -> LinkResultList[_M] | None: ... + + def __getitem__(cls, item: Any) -> Any: + return cls.oold_query(item) + + +_UNION_ORIGINS = {Union} +if hasattr(types, "UnionType"): # PEP 604: X | None + _UNION_ORIGINS.add(types.UnionType) + + +def _extract_target(annotation: Any) -> tuple[Any, bool, bool]: + """Return (target_type, is_many, optional) for a link annotation. + + Understands ``Optional[List[X]]`` and the ``Link[X]`` / ``LinkList[X]`` form, + where the to-many-ness comes from the class rather than a surrounding + ``list``. + + ``optional`` says whether a missing value is a legitimate answer. Only the + ``Link[...]`` form can say no: writing ``Link[Person]`` rather than + ``Link[Person | None]`` declares the link mandatory, and the binding then + keeps that promise instead of handing back a ``None`` the type denies. Every + other spelling stays optional, so existing declarations are unaffected. + """ + many = False + seen_link_annotation = False + saw_none_arm = False + target = annotation + changed = True + while changed: + changed = False + origin = get_origin(target) + if isinstance(origin, type) and issubclass(origin, _LinkAnnotation): + args = get_args(target) + if args: + target, changed = args[0], True + many = many or origin._many + seen_link_annotation = True + elif origin in _UNION_ORIGINS: + # Order-independent: Optional[Link[T]] meets the union first and + # Link[T | None] meets it second, and both mean the same thing. An + # earlier version only looked once a Link had been seen, so the + # outer-Optional spelling came out as its opposite - mandatory. + if type(None) in get_args(target): + saw_none_arm = True + args = [a for a in get_args(target) if a is not type(None)] + if len(args) == 1: + target, changed = args[0], True + elif origin is list: + args = get_args(target) + if args: + target, many, changed = args[0], True, True + # Only the Link[...] form can declare a link mandatory, and only when no + # None arm appears anywhere in the annotation. + optional = not seen_link_annotation or saw_none_arm + return target, many, optional + + +def _is_link_annotation(annotation: Any) -> bool: + """Whether ``Link[...]`` / ``LinkList[...]`` appears anywhere in an annotation. + + Lets the annotation alone declare a link, so ``knows: LinkList["Person"]`` + needs no keyword in ``json_schema_extra``. + """ + seen: list[Any] = [annotation] + while seen: + current = seen.pop() + origin = get_origin(current) + if isinstance(origin, type) and issubclass(origin, _LinkAnnotation): + return True + seen.extend(get_args(current)) + return False + + +_TYPE_REGISTRY: dict[str, type] = {} +"""Maps a ``type`` field default (the class IRI) to its model class. + +Identity matters, not just contents. Downstream imports the shipped registry +directly and **writes into it**:: + + from oold.model import _types + _types[SomeClass.get_cls_iri()] = SomeClass + +so on integration this must *be* ``oold.model._types``, not a second dict - +otherwise those registrations are invisible here and polymorphic resolution +silently falls back to the declared target. Use :func:`use_type_registry`. +""" + + +def use_type_registry(registry: dict) -> None: + """Adopt an existing registry mapping, sharing its identity. + + Call with ``oold.model._types`` when this binding replaces the shipped one, + so registrations made through either name are seen by both. + """ + global _TYPE_REGISTRY + _TYPE_REGISTRY = registry + + +def _resolve_cls(data: dict[str, Any], target: Any) -> Any: + """Pick the most specific class for a document, by its type IRI.""" + type_iri = data.get("type") + if isinstance(type_iri, list): + type_iri = type_iri[0] if type_iri else None + if isinstance(type_iri, str): + found = _TYPE_REGISTRY.get(type_iri) + if found is not None: + return found + return target + + +class LinkResultList(list[T]): + """List returned by a to-many link. + + Adds IRI lookup, filtering and attribute projection, and keeps mutations in + sync with the owner's link storage: appending or removing an item updates + the stored references too, so ``__iris__`` and serialisation stay correct + without a second write. + + Generic in the item type, so ``Entity[cond][0]`` is an ``Entity`` to a type + checker rather than ``Any``. Whether an element may be ``None`` is declared: + ``LinkList[Person]`` promises every reference resolves and raises if one does + not, ``LinkList[Person | None]`` keeps the slot as ``None``. The slot is kept + either way, so the list stays aligned with the stored references. + """ + + _owner: Any = None + _field: str | None = None + _refs: list[Any] | None = None + + def _bind(self, owner: Any, field: str, refs: Any = None) -> LinkResultList: + self._owner = owner + self._field = field + # the references this list was built from, so an entry that could not be + # resolved can be written back as the reference it still is + self._refs = list(refs) if refs else [] + return self + + def _sync(self) -> None: + if self._owner is None or self._field is None: + return + # A slot that did not resolve reads as None, but dropping it would + # delete the reference from storage - the list would shrink and the IRI + # be lost. `_refs` is kept positionally aligned with this list by every + # mutator below, so the reference for a None slot is the one at the same + # index. Matching them up in order instead deleted the wrong element. + refs = self._refs or [] + values = [] + for index, value in enumerate(self): + if value is not None: + values.append(value) + continue + ref = refs[index] if index < len(refs) else None + if ref is not None: + values.append(ref) + descr = type(self._owner).__link_fields__[self._field] + descr.set_value(self._owner, values) + # keep the cached read pointing at this very list + self._owner.__dict__[self._field] = self + + # Every mutating operation syncs, and applies the same structural change to + # _refs so the two stay aligned. Covering only append/remove/extend left + # `links[0] = x`, `pop()`, `insert()`, `clear()`, `del` and `+=` changing + # what you see while storage kept the old references. + def _refs_list(self) -> list: + if self._refs is None: + self._refs = [] + return self._refs + + def append(self, item: Any) -> None: + super().append(item) + self._refs_list().append(None) + self._sync() + + def remove(self, item: Any) -> None: + index = self.index(item) + super().remove(item) + refs = self._refs_list() + if index < len(refs): + refs.pop(index) + self._sync() + + def extend(self, iterable: Any) -> None: + items = list(iterable) + super().extend(items) + self._refs_list().extend([None] * len(items)) + self._sync() + + def insert(self, index: SupportsIndex, item: Any) -> None: + super().insert(index, item) + self._refs_list().insert(index, None) + self._sync() + + def pop(self, index: SupportsIndex = -1) -> Any: + item = super().pop(index) + refs = self._refs_list() + with contextlib.suppress(IndexError): + refs.pop(index) + self._sync() + return item + + def clear(self) -> None: + super().clear() + self._refs_list().clear() + self._sync() + + def sort(self, **kwargs: Any) -> None: + # order becomes unknowable for unresolved slots, so drop their refs + # rather than pair them with the wrong element + super().sort(**kwargs) + self._refs = [None] * len(self) + self._sync() + + def reverse(self) -> None: + super().reverse() + self._refs_list().reverse() + self._sync() + + def __setitem__(self, index: Any, value: Any) -> None: + super().__setitem__(index, value) + refs = self._refs_list() + if isinstance(index, slice): + refs[index] = [None] * len(self[index]) + elif index < len(refs): + refs[index] = None + self._sync() + + def __delitem__(self, index: Any) -> None: + super().__delitem__(index) + refs = self._refs_list() + with contextlib.suppress(IndexError): + del refs[index] + self._sync() + + def __iadd__(self, other: Any) -> LinkResultList: + items = list(other) + super().__iadd__(items) + self._refs_list().extend([None] * len(items)) + self._sync() + return self + + @overload + def __getitem__(self, index: Condition | bool) -> LinkResultList[T]: ... + + @overload + def __getitem__(self, index: SupportsIndex) -> T: ... + + @overload + def __getitem__(self, index: slice) -> LinkResultList[T]: ... + + @overload + def __getitem__(self, index: str) -> Any: ... + + def __getitem__( # pyright: ignore[reportIncompatibleMethodOverride] + self, index: Any + ) -> Any: + if isinstance(index, str): + if index.startswith("@"): + # inline query form: links["@name=='Entity 2'"] + key, _, raw = index[1:].partition("==") + wanted = raw.strip().strip("'\"") + return LinkResultList( + item for item in self if item is not None and getattr(item, key.strip(), None) == wanted + ) + for item in self: + if item is not None and getattr(item, "id", None) == index: + return item + raise KeyError(index) + if isinstance(index, Condition): + return LinkResultList( + item + for item in self + if item is not None and apply_operator(index.operator, getattr(item, index.field, None), index.value) + ) + return list.__getitem__(self, index) + + def __getattr__(self, name: str) -> Any: + # Only invoked when normal lookup fails, so list methods are unaffected. + if name.startswith("_"): + raise AttributeError(name) + out = LinkResultList() + for item in self: + if item is None: + continue + value = getattr(item, name) + if isinstance(value, list): + out.extend(value) + else: + out.append(value) + return out + + +def _batch_resolve(refs: list[Ref | None], target: Any) -> list[Any]: + """Resolve all unresolved refs, one backend call per resolver prefix.""" + pending = [r for r in refs if r is not None and r._obj is None and r.iri] + groups: dict[str, list[Ref]] = defaultdict(list) + for r in pending: + groups[r.iri.split(":")[0]].append(r) + for group in groups.values(): + iris = [r.iri for r in group] + resolver = get_resolver(GetResolverParam(iri=iris[0])).resolver + # Go through resolve(), not resolve_iris(): it applies the backend's + # format (a JSON-LD store hands back expanded JSON-LD, which cannot be + # fed to the model directly) and dispatches on the document's type IRI, + # so a stored subclass resolves to the subclass. + # + # A union target (LinkList["Person | Org"]) is not a class, so it cannot + # be a model_cls. Hand over the root model instead and let the same type + # dispatch pick the arm - passing the union made ResolveParam validation + # fail, which used to drop into the fallback below and construct the raw + # document, i.e. every union link was broken on every JSON-LD backend. + model_cls = target if isinstance(target, type) else LinkedBaseModel + try: + nodes = resolver.resolve(ResolveParam(iris=iris, model_cls=model_cls)).nodes + except NotImplementedError: + # A backend that does not implement resolve() at all: fall back to + # the raw documents. Deliberately narrow - catching everything here + # turned a malformed document, or any error inside from_jsonld, into + # a second request whose result was then built against the declared + # target, losing the original error and silently mis-constructing. + fetched = resolver.resolve_iris(iris) + nodes = { + iri: (_construct(_resolve_cls(d, target), d) if d is not None else None) for iri, d in fetched.items() + } + for r in group: + r._obj = nodes.get(r.iri) + return [None if r is None else r._obj for r in refs] + + +def _to_ref(value: Any, target: Any) -> Ref | None: + if value is None: + return None + if isinstance(value, Ref): + if value._target is None: + value._target = target + return value + if isinstance(value, str): + return Ref(iri=value, target=target) + if isinstance(value, dict): + # construct through the model so the linked object is validated + cls = _resolve_cls(value, target) + if cls is None: + raise ValueError(f"Cannot construct link from {value!r}: unknown target") + return Ref(obj=_construct(cls, value), target=target) + return Ref(obj=value, target=target) + + +def _register_class(cls: type) -> None: + """Register a class under the type IRIs it introduces. + + Mirrors the shipped metaclass: controllers are collected in a separate + table so they never shadow the data model they extend, and a data model may + only claim the IRIs it introduces itself - a subclass that merely narrows a + field reports its parent's IRI and would otherwise replace it. + """ + from oold.model import _controller_types, _inherited_cls_iris + + iri = cls.get_cls_iri() if hasattr(cls, "get_cls_iri") else None + if iri is None: + return + is_ctrl = any(b.__module__ == "oold.model" and b.__name__ == "BaseController" for b in cls.__mro__) + inherited = frozenset() if is_ctrl else _inherited_cls_iris(cls) + for value in iri if isinstance(iri, list) else [iri]: + if not isinstance(value, str): + continue + if is_ctrl: + _controller_types.setdefault(value, []).append(cls) + elif value not in inherited: + _TYPE_REGISTRY[value] = cls + + +class _AutoLink: + """Data descriptor backing a link field. + + Installed automatically for annotated ``x-oold-range`` fields (implicit + form), or declared directly in a class body via :class:`Link` / + :class:`LinkList` (explicit form). Both forms share this implementation, so + runtime behaviour is identical. + """ + + def __init__( + self, + name: str | None = None, + target: Any = None, + many: bool = False, + optional: bool = True, + required_iri: bool = False, + ): + self.name = name + self.target = target + self.many = many + # False only for the Link[T] / LinkList[T] form without a None arm + self.optional = optional + # x-oold-required-iri: the schema says this link must carry a reference + self.required_iri = required_iri + self.owner: Any = None + + def __set_name__(self, owner: type, name: str) -> None: + # Only relevant for the explicit form (declared in the class body). + if self.name is None: + self.name = name + self.owner = owner + + def _target_cls(self, owner: Any) -> Any: + cached = self.__dict__.get("_resolved_target") + if cached is not None: + return cached + target = self.target + if target is None: + # Explicit form declared as LinkList["Person"]() with no argument: + # recover the type argument from __orig_class__, which typing sets + # on the instance after __init__ (also inside a class body). + orig = self.__dict__.get("__orig_class__") + if orig is not None: + args = get_args(orig) + if args: + target = args[0] + if hasattr(target, "__forward_arg__"): # ForwardRef("Person") + target = target.__forward_arg__ + if isinstance(target, str): + import sys + + module = sys.modules.get(getattr(owner or self.owner, "__module__", ""), None) + target = getattr(module, target, None) if module else None + if target is not None: + self.__dict__["_resolved_target"] = target + return target + + def range_iri(self, owner: Any = None) -> Any: + """The target's schema IRI, for deriving ``x-oold-range``. + + ``Link[T]`` already names the target, so repeating it in + ``OoldField(range=...)`` states the same thing twice and lets the two + disagree. The schema is derived from the annotation instead. + """ + target = self._target_cls(owner) + get_iri = getattr(target, "get_cls_iri", None) + if get_iri is None: + return None + try: + return get_iri() or None + except Exception: + # a target that cannot name itself simply contributes no range + return None + + def __get__(self, obj: Any, objtype: Any = None) -> Any: + if obj is None: + if _Constructing.is_active(): + # A subclass may redeclare an inherited link field. Pydantic + # checks the bases for a same-named attribute and rejects the + # field if it finds one, so the descriptor has to stay invisible + # while a class is being built - same reason the metaclass + # carries the flag. + raise AttributeError(self.name) + # Class access returns the descriptor, so Person.knows == "x" can + # build a Condition without any metaclass involvement. + return self + stored = obj._links.get(self.name) + target = self._target_cls(objtype or type(obj)) + if self.many: + items = _batch_resolve(stored, target) if stored else [] + if not self.optional and any(item is None for item in items): + # the declaration promised every element resolves + missing = [r.iri for r, item in zip(stored, items, strict=False) if item is None] + raise LinkNotResolved(self._message(obj, missing)) + result = LinkResultList(items)._bind(obj, self.name, stored) + elif stored is None: + if not self.optional: + # Raised on access, not at construction. The annotation says what + # *reading* the link yields, not that every instance carries one: + # graph data is routinely partial, and rejecting such objects when + # they are built would make them unloadable. Declaring the link + # mandatory states an intent to traverse it, so one try/except + # around a whole chain replaces a guard at every hop. + raise LinkNotResolved( + f"{type(obj).__name__}.{self.name} is declared mandatory but is " + f"not set. Declare it as Link[T | None] if absence is data." + ) + result = None + else: + result = _batch_resolve([stored], target)[0] + if result is None and not self.optional: + raise LinkNotResolved(self._message(obj, [stored.iri])) + # Store the resolved value in the instance __dict__. This descriptor is + # deliberately NON-data (no __set__), so from now on normal attribute + # lookup finds the instance dict first and never calls back into Python: + # warm link reads run at native speed (the functools.cached_property + # pattern). Writes are still intercepted, by LinkedModel.__setattr__. + obj.__dict__[self.name] = result + return result + + def _message(self, obj: Any, iris: list[Any]) -> str: + listed = ", ".join(str(iri) for iri in iris if iri) + return ( + f"{type(obj).__name__}.{self.name} is declared mandatory, but " + f"{listed or 'the reference'} could not be resolved. The backend " + f"answered without it - declare the link as optional if that is a " + f"legitimate answer." + ) + + def set_value(self, obj: Any, value: Any) -> None: + target = self._target_cls(type(obj)) + if self.many: + obj._links[self.name] = [] if value is None else [_to_ref(v, target) for v in value] + else: + obj._links[self.name] = _to_ref(value, target) + obj.__dict__.pop(self.name, None) # invalidate the cached read + + def __eq__(self, other: Any) -> Any: # type: ignore[override] + return Condition(field=self.name, operator="eq", value=other) + + def __ne__(self, other: Any) -> Any: # type: ignore[override] + return Condition(field=self.name, operator="ne", value=other) + + def __hash__(self) -> int: + return id(self) + + def iris(self, obj: Any) -> Any: + stored = obj._links.get(self.name) + if self.many: + return [r.iri for r in (stored or []) if r is not None and r.iri] + return stored.iri if stored is not None else None + + +class _LinkAnnotation: + """Lets ``Link[T]`` / ``LinkList[T]`` stand in for the target annotation. + + A link has two types, and one annotation cannot state both: what you read is + a resolved object, what you may write is that object *or* a reference to it + (an IRI string, or a JSON object still to be constructed). Declaring the + field as the descriptor type is what carries both - a type checker takes the + ``__init__`` parameter and the assignment type from ``__set__`` and the + attribute type from ``__get__`` (PEP 681). + + Pydantic is told to build the schema of the *target* instead, so the emitted + JSON Schema is byte-identical to the plain annotation - ``$ref``, arrays and + unions included - and forward references still resolve on ``model_rebuild``. + """ + + _many: ClassVar[bool] = False + + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> Any: + args = get_args(source_type) + target = args[0] if args else Any + return handler(list[target] if cls._many else target) + + +class Link(_AutoLink, _LinkAnnotation, Generic[T]): + """A to-one link. The recommended way to declare one. + + Two spellings, not equivalent to a type checker:: + + employer: Link[Organization] = OoldField() # recommended + employer = Link(Organization) # not a pydantic field + + The annotation form types both directions: it reads as ``T`` and accepts a + ``T``, an IRI or a JSON object on assignment. ``Link[T]`` promises a ``T``, + so a chain needs no guard per hop; write ``Link[T | None]`` where absence is + part of the model. Nested - ``Optional[Link[T]]`` - the read type still + narrows but the write type does not, so spell the union inside. + """ + + _many: ClassVar[bool] = False + + def __init__(self, target: type[T] | str | None = None): + super().__init__(name=None, target=target, many=False) + + @overload + def __get__(self, obj: None, objtype: Any = None) -> Link[T]: ... + + @overload + def __get__(self, obj: object, objtype: Any = None) -> T: ... + + def __get__(self, obj: Any, objtype: Any = None) -> Any: + return _AutoLink.__get__(self, obj, objtype) + + if TYPE_CHECKING: + # Declared for the checker only. At runtime this stays a **non-data** + # descriptor, so the instance __dict__ keeps shadowing it after the first + # read - which is what makes a warm link read cost the same as a plain + # field. Writes are intercepted by LinkedModel.__setattr__ instead, which + # applies exactly the conversion declared here. + def __set__(self, obj: object, value: T | str | Mapping[str, Any] | None) -> None: ... + + +class LinkList(_AutoLink, _LinkAnnotation, Generic[T]): + """A to-many link. The recommended way to declare one. + + Two spellings, not equivalent to a type checker:: + + knows: LinkList["Person"] = OoldField() # recommended + knows = LinkList("Person") # not a pydantic field + + The annotation form reads as ``LinkResultList[T]`` - never ``None`` itself, + an unset link is an empty list - and accepts objects, IRIs or JSON objects + on assignment. Prefer it to ``list[Link[T]]``, which a checker reads as a + list of descriptors. + """ + + _many: ClassVar[bool] = True + + def __init__(self, target: type[T] | str | None = None): + super().__init__(name=None, target=target, many=True) + + @overload + def __get__(self, obj: None, objtype: Any = None) -> LinkList[T]: ... + + @overload + def __get__(self, obj: object, objtype: Any = None) -> LinkResultList[T]: ... + + def __get__(self, obj: Any, objtype: Any = None) -> Any: + return _AutoLink.__get__(self, obj, objtype) + + if TYPE_CHECKING: + # see Link.__set__ - checker-only, so the descriptor stays non-data + def __set__(self, obj: object, value: Iterable[T | str | Mapping[str, Any]] | None) -> None: ... + + +def _excluded(info: Any, name: str) -> bool: + """Whether the caller asked for an unset link to be left out.""" + if getattr(info, "exclude_none", False): + return True + if getattr(info, "exclude_unset", False) or getattr(info, "exclude_defaults", False): + return True + exclude = getattr(info, "exclude", None) + return bool(exclude) and name in exclude + + +def _alias_strings(alias: Any) -> list[str]: + """Every name an alias can be given under. + + ``validation_alias`` is not always a string: ``AliasChoices`` holds several, + and each may itself be an ``AliasPath``. Accepting only ``str`` left those + payload keys for pydantic to validate against the *target* model. + """ + if isinstance(alias, str): + return [alias] + choices = getattr(alias, "choices", None) + if choices is not None: + out = [] + for choice in choices: + out.extend(_alias_strings(choice)) + return out + path = getattr(alias, "path", None) + if path and isinstance(path[0], str): + return [path[0]] + return [] + + +def _link_aliases(cls: type) -> dict[str, str]: + """alias -> field name, for link fields that declare one. + + Computed once per class in ``__pydantic_init_subclass__`` - it was rebuilt + on every construction, which cost about a quarter of the time to build an + object. + """ + out: dict[str, str] = {} + for name in getattr(cls, "__link_fields__", {}): + field = cls.model_fields.get(name) + for alias in (getattr(field, "validation_alias", None), getattr(field, "alias", None)): + for text in _alias_strings(alias): + out[text] = name + return out + + +def _emit_inline(stored: Any) -> Any: + """Serialise references that have no IRI, so they are not silently lost.""" + + def one(ref: Any) -> Any: + obj = getattr(ref, "_obj", None) if ref is not None else None + if obj is None: + return None + return obj.model_dump(exclude_none=True) if hasattr(obj, "model_dump") else obj + + if isinstance(stored, list): + return [one(r) for r in stored] + return one(stored) + + +class LinkedBaseModel(BaseModel, LinkedApiMixin, metaclass=LinkedBaseModelMetaClass): + """Base model supporting both implicit and explicit link declarations.""" + + model_config = ConfigDict(ignored_types=(Link, LinkList, _AutoLink)) + + _links: dict[str, Any] = PrivateAttr(default_factory=dict) + # references assigned through __iris__ for names that are not link fields; + # the shipped side-dict kept them, so reading them back has to work + _extra_iris: dict[str, Any] = PrivateAttr(default_factory=dict) + __link_fields__: ClassVar[dict[str, _AutoLink]] = {} + __link_aliases__: ClassVar[dict[str, str]] = {} + __required_links__: ClassVar[tuple[str, ...]] = () + __link_defaults__: ClassVar[dict[str, Any]] = {} + + @classmethod + def oold_query(cls, item: Any) -> Any: + """Resolve ``Model[...]`` against every registered resolver. + + A single IRI yields one instance, a list or a condition yields a list. + Resolvers that cannot answer a structured query are skipped. + """ + node_list: list = [] + for resolver in interface._resolvers.values(): + try: + if isinstance(item, (str, list)): + nodes = resolver.resolve( + ResolveParam( + iris=[item] if isinstance(item, str) else item, + model_cls=cls, + ) + ).nodes.values() + else: + nodes = resolver.query(QueryParam(query=item, model_cls=cls)).nodes.values() + node_list.extend(nodes) + except NotImplementedError: + continue + # A query answers with what it found. An IRI the backend cannot place is + # not a match, and keeping a None for it would contradict the element + # type - unlike a to-many link, there is no declaration here promising + # the result stays aligned with anything. + node_list = [node for node in node_list if node is not None] + if isinstance(item, str): + return node_list[0] if node_list else None + return LinkResultList(node_list) if node_list else None + + @classmethod + def __get_pydantic_json_schema__(cls, core_schema_: Any, handler: Any) -> Any: + """Write ``x-oold-range`` for links that only stated it in the annotation. + + Presence of ``x-oold-range`` is what makes a property a link, so a + schema carrying only ``x-oold-link`` does not round-trip through code + generation. ``Link[T]`` names the target already, so the keyword is + derived from it rather than repeated in ``OoldField(range=...)``. + + Done here, not when the descriptor is installed: a forward reference is + not resolvable at class-creation time, and a ``Field()`` object shared + between models must not be mutated in place. + """ + schema = handler(core_schema_) + try: + schema = handler.resolve_ref_schema(schema) + except Exception: + return schema + properties = schema.get("properties") if isinstance(schema, dict) else None + if not properties: + return schema + link_fields = cls.__link_fields__ + aliases = cls.__link_aliases__ + required = schema.get("required") + for key, prop in properties.items(): + name = key if key in link_fields else aliases.get(key) + descr = link_fields.get(name) if name else None + if descr is None or not isinstance(prop, dict): + continue + if descr.required_iri: + # A link is never required at the pydantic level - its value is + # routed out of the payload before validation - so pydantic + # leaves it out of `required`. Stating it only in + # x-oold-required-iri would hide the constraint from every + # plain JSON Schema validator. + if required is None: + required = schema["required"] = [] + if key not in required: + required.append(key) + if prop.get("x-oold-range") or prop.get("range"): + continue + iri = descr.range_iri(cls) + if iri: + prop["x-oold-range"] = iri + # the range says "link" on its own; the marker was a stand-in + prop.pop("x-oold-link", None) + return schema + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + super().__pydantic_init_subclass__(**kwargs) + if not links_enabled(): + # Plain-pydantic mode: install nothing. Link fields keep the + # semantics their annotation already states - a nested model, not + # an IRI reference - so the class behaves exactly like a plain + # BaseModel without touching the declaration. + cls.__link_fields__ = {} + return + links: dict[str, _AutoLink] = dict(getattr(cls, "__link_fields__", {})) + # Explicit form: descriptors declared directly in the class body. + for klass in reversed(cls.__mro__): + for key, value in vars(klass).items(): + if isinstance(value, _AutoLink): + links[key] = value + # Implicit form: annotated fields carrying a range keyword. + for name, field in cls.model_fields.items(): + extra = field.json_schema_extra + extra = extra if isinstance(extra, dict) else {} + rng = extra.get("x-oold-range", extra.get("range")) + # x-oold-link marks a link whose target comes from the annotation; + # a Link[...] / LinkList[...] annotation says the same on its own + if not rng and not extra.get("x-oold-link") and not _is_link_annotation(field.annotation): + continue + target, many, optional = _extract_target(field.annotation) + if isinstance(rng, str) and not isinstance(target, type): + target = rng + descr = _AutoLink(name, target, many, optional, bool(extra.get("x-oold-required-iri"))) + setattr(cls, name, descr) + links[name] = descr + cls.__link_fields__ = links + # per-class constants, so construction does not recompute them + cls.__link_aliases__ = _link_aliases(cls) + cls.__required_links__ = tuple(n for n, d in links.items() if d.required_iri) + _register_class(cls) + + def __init__(self, *args: Any, **data: Any) -> None: + # The shipped model accepts another model as the first positional + # argument as a cast shorthand: Target(source, extra="value"). + if args and isinstance(args[0], BaseModel): + source = args[0] + base = source._raw_dict() if hasattr(source, "_raw_dict") else source.model_dump() + base.pop("type", None) + data = {**{k: v for k, v in base.items() if v is not None}, **data} + elif args: + raise TypeError(f"{type(self).__name__}() takes no positional arguments other than a source model") + link_fields = type(self).__link_fields__ + # Route link values out of the payload before pydantic validates - by + # field name and by alias, since a payload built with by_alias=True uses + # the alias and would otherwise be validated against the target model. + aliases = type(self).__link_aliases__ + link_data = {} + for key in list(data): + name = key if key in link_fields else aliases.get(key) + if name is not None: + link_data[name] = data.pop(key) + super().__init__(**data) + # Pydantic writes each field's default into __dict__, and an entry there + # shadows a non-data descriptor - so an unset link would keep returning + # that default (None) and never reach __get__. Dropping the entries hands + # unset links back to the descriptor, which answers [] for to-many and + # None for to-one. That is what makes a non-Optional list annotation + # truthful rather than a lie about a value that is really None. + for _name in link_fields: + self.__dict__.pop(_name, None) + # A link field's pydantic default is stripped, so seed the declared IRI + # here - the link then resolves lazily like any other, instead of the + # default being lost or fetched on every construction. + for _name, _iris in type(self).__link_defaults__.items(): + if _name in link_fields and _name not in link_data: + link_data[_name] = _iris + for key, value in link_data.items(): + self._set_link(key, value) + missing = [name for name in type(self).__required_links__ if not self._links.get(name)] + if missing: + # x-oold-required-iri, enforced as the legacy binding did. It raised + # on the mere presence of the keyword; this raises on a true value, + # so required_iri=False no longer means "required". + raise ValueError(f"{', '.join(sorted(missing))} is required but not set") + + def _set_link(self, name: str, value: Any) -> None: + """Store one supplied link value. + + The hook a notation overrides to interpret the value - a union arm has + to decide literal from reference. Doing it here rather than after + ``__init__`` returns is what lets the required-link check see the links + a subclass sets: it ran before them, and reported every required link of + a notation model as missing. + """ + type(self).__link_fields__[name].set_value(self, value) + + def __eq__(self, other: Any) -> bool: + """Compare by data, not by what happens to be cached. + + Resolving a link stores the resolved object in ``__dict__`` (that is + what makes warm reads native-speed), and pydantic's ``__eq__`` compares + ``__dict__`` - so reading a link would otherwise change the result of a + comparison. Links are compared by their stored references instead, and + the remaining fields the normal way. + """ + if other.__class__ is not self.__class__: + return NotImplemented + # Compare the same state pydantic does - extras and private attributes + # included. Looking at __dict__ alone made two models with different + # extra="allow" fields compare equal. + if self.__pydantic_extra__ != other.__pydantic_extra__: + return False + if self.__pydantic_private__ != other.__pydantic_private__: + return False + links = type(self).__link_fields__ + if links: + mine = {k: v for k, v in self.__dict__.items() if k not in links} + theirs = {k: v for k, v in other.__dict__.items() if k not in links} + if mine != theirs: + return False + return all(links[name].iris(self) == links[name].iris(other) for name in links) + return self.__dict__ == other.__dict__ + + __hash__ = None # type: ignore[assignment] + """Unhashable, as pydantic models are. + + An earlier ``__hash__ = id(self)`` made models hashable, so ``set(models)`` + deduplicated by identity instead of raising - silently different from both + the legacy binding and plain pydantic. + """ + + def __setattr__(self, name: str, value: Any, internal: bool = False) -> None: + # internal=True means "write the value as given": BaseController passes + # it through to bypass link handling for controller-only state. + if name == "__iris__": + # a property with a setter on the mixin - pydantic would otherwise + # reject it as "no field __iris__" + LinkedApiMixin.__iris__.fset(self, value) + return + if internal: + super().__setattr__(name, value) + return + # Targeted: only link names are routed to the descriptor. Needed because + # pydantic's own __setattr__ writes model fields straight into __dict__, + # bypassing a data descriptor's __set__ (which would leave the link + # storage and its cache stale). Every other write stays native, and + # BaseModel already defines __setattr__, so this adds no new slot cost. + descr = type(self).__link_fields__.get(name) + if descr is not None: + descr.set_value(self, value) + else: + super().__setattr__(name, value) + + @model_serializer(mode="wrap") + def _serialize_links(self, handler: Any, info: SerializationInfo) -> dict[str, Any]: + # Reading a link caches the resolved object in __dict__, where pydantic's + # own serializer then finds it and serialises it as the declared type. + # For a to-many link that could not be fully resolved the cache holds a + # None among the objects, and `list[Bar]` has no way to render it: + # serialising after such a read died with "type object 'NoneType' has no + # attribute 'model_fields'". The link keys are replaced below in any + # case, so the cache is hidden from the handler rather than repaired. + cached = {} + for _name in type(self).__link_fields__: + if _name in self.__dict__: + cached[_name] = self.__dict__.pop(_name) + try: + d = handler(self) + finally: + self.__dict__.update(cached) + fields = type(self).model_fields + by_alias = bool(getattr(info, "by_alias", False)) + for name, descr in type(self).__link_fields__.items(): + # honour by_alias: every other key does, so writing the link under + # its field name produced a payload mixing both spellings. The key + # is never in `d` to compare against - link values are routed out of + # __dict__ - so the decision comes from the serialisation context. + name_out = name + if by_alias: + field = fields.get(name) + alias = getattr(field, "serialization_alias", None) or getattr(field, "alias", None) + if isinstance(alias, str): + name_out = alias + iris = descr.iris(self) + if iris: + d.pop(name, None) + d[name_out] = iris + continue + stored = self._links.get(name) + if stored is None: + # No value: never set, or explicitly cleared with `= None`. + # Those are the same statement, and the legacy binding emits + # neither - so distinguishing them left an explicit null behind + # after a caller had cleared the link. + # + # The key holds None, as the legacy binding does, but only when + # the caller has not asked for exactly this to be left out. + # Writing it unconditionally runs *after* handler() has applied + # the exclusions, which would leak an explicit null past + # exclude_none, exclude_unset, exclude_defaults and + # exclude={...} into every stored document. + d.pop(name, None) + if not _excluded(info, name): + d[name_out] = None + continue + # Set, but nothing to reference: either an explicit empty list - a + # different statement from "unset" and one that must round-trip - or + # an inline object with no IRI, which has to serialise nested rather + # than vanish, since cast() is built on this. + d.pop(name, None) + d[name_out] = _emit_inline(stored) + return d + + +# Downstream subclasses this metaclass by name, so the name is public API and +# must stay bound to whatever metaclass LinkedBaseModel actually uses. +LinkedQueryMeta = LinkedBaseModelMetaClass diff --git a/src/oold/model/_notation.py b/src/oold/model/_notation.py new file mode 100644 index 0000000..fb965b9 --- /dev/null +++ b/src/oold/model/_notation.py @@ -0,0 +1,270 @@ +"""The link declaration notations proposed in issue #107 review comments. + +1. ``OoldField()`` / ``OoldField(link=True)`` - no ``range=`` argument. The link + target is inferred from the annotation, so the schema IRI is not repeated in + Python. Note the trade-off: nothing then writes ``x-oold-range`` into the + emitted schema, so pass ``range=`` where the schema is the artifact. +2. ``Link[T]`` / ``LinkList[T]`` as the **whole** annotation, e.g. + ``employer: Link[Organization]`` or ``knows: LinkList["Person"]``. These are + descriptor types, so a checker takes the ``__init__`` parameter and the + assignment type from ``__set__`` and the attribute type from ``__get__`` + (PEP 681) - which is how one field carries both the resolved read type and + the IRI-or-object write type. Optionality is declared in the parameter: + ``Link[T]`` reads as ``T``, ``Link[T | None]`` as ``T | None``. + + They must be the whole annotation. Nested - ``list[Link[T]]`` or + ``Optional[Link[T]]`` - a checker does not apply descriptor rules and the + read type comes back wrong; use ``LinkList[T]`` and ``Link[T | None]``. +3. **Union forms** mixing literal, inline object and reference, e.g. + ``location: Union[str, Location, None] = OoldField(link=True)``. + +Everything reuses the descriptor machinery from +:mod:`oold.model._descriptor`; ``Link`` and ``LinkList`` are re-exported from +there rather than redefined, so there is one implementation, not two. +""" + +from __future__ import annotations + +import types +from typing import ( + Annotated, + Any, + ClassVar, + TypeVar, + Union, + get_args, + get_origin, +) + +from pydantic import BaseModel, model_serializer + +from oold.model._descriptor import ( + Link, + LinkedBaseModel, + LinkList, + OoldField, + _AutoLink, + _extract_target, + _LinkAnnotation, + _register_class, +) + +# Link and LinkList are re-exported: the notation module is the documented entry +# point for these declarations, and they are one implementation, not two. +__all__ = [ + "Link", + "LinkList", + "OoldField", + "OoldModel", +] + +T = TypeVar("T") + +_LITERAL_TYPES = (str, int, float, bool, bytes) + + +_UNION_ORIGINS = {Union} +if hasattr(types, "UnionType"): # PEP 604: X | None + _UNION_ORIGINS.add(types.UnionType) + + +def _unwrap(annotation: Any) -> tuple[Any, bool, bool, list[Any]]: + """Return (target, many, has_link_marker, literal_arms) for an annotation. + + Understands ``Optional[...]``, ``List[...]``, ``Annotated[...]`` and unions + mixing a literal arm, an inline-object arm and a ``Link[...]`` arm. + """ + many = False + marked = False + literals: list[Any] = [] + target = annotation + + def strip(tp: Any) -> Any: + nonlocal marked, many + while True: + origin = get_origin(tp) + if origin is Annotated: + tp = get_args(tp)[0] + continue + # Link[X] / LinkList[X]: the annotation itself declares the link, + # and LinkList carries the to-many-ness instead of a list wrapper + if isinstance(origin, type) and issubclass(origin, _LinkAnnotation): + args = get_args(tp) + if not args: + break + marked = True + many = many or origin._many + tp = args[0] + continue + break + return tp + + changed = True + while changed: + changed = False + target = strip(target) + origin = get_origin(target) + if origin in _UNION_ORIGINS: + arms = [a for a in get_args(target) if a is not type(None)] + model_arms, other = [], [] + for arm in arms: + bare = strip(arm) + if isinstance(bare, type) and issubclass(bare, BaseModel): + model_arms.append(bare) + elif bare in _LITERAL_TYPES: + other.append(bare) + else: + model_arms.append(bare) + literals.extend(other) + if len(model_arms) >= 1: + target, changed = model_arms[0], True + elif other: + target, changed = other[0], True + elif origin in (list, list): + args = get_args(target) + if args: + target, many, changed = strip(args[0]), True, True + return target, many, marked, literals + + +def _emit_one(ref: Any, boxed: bool) -> Any: + """Serialise a single stored reference. + + ``boxed`` is set when the field also accepts a literal, in which case a + reference must be written as ``{"@id": ...}`` so that re-reading it cannot + be confused with text. A value without an IRI has no reference to emit, so + it is written inline - a blank node. + """ + if ref is None: + return None + iri = getattr(ref, "iri", None) + if iri: + return {"@id": iri} if boxed else iri + obj = getattr(ref, "_obj", None) + if obj is None: + return None + return obj.model_dump(exclude_none=True) if hasattr(obj, "model_dump") else obj + + +def _emit(stored: Any, boxed: bool) -> Any: + if isinstance(stored, list): + out = [_emit_one(r, boxed) for r in stored] + return [v for v in out if v is not None] + return _emit_one(stored, boxed) + + +class OoldModel(LinkedBaseModel): + """Model base supporting the proposed link notations. + + Subclasses the binding rather than re-implementing it: it was a bare + ``BaseModel``, so it was not a ``GenericLinkedBaseModel`` and could not be + passed as a ``model_cls``. Resolution therefore always failed validation and + only worked through ``_batch_resolve``'s fallback - which is to say, through + the error path - and ``oold_query`` was a stub returning a tuple. It also + carried byte-identical copies of ``__eq__``, ``__hash__``, ``get_iri`` and + ``link_iris``. + + What stays here is the part that genuinely differs: union arms, where a bare + string is a literal rather than a reference. + """ + + __link_literals__: ClassVar[dict[str, list[Any]]] = {} + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + super().__pydantic_init_subclass__(**kwargs) + links: dict[str, _AutoLink] = dict(getattr(cls, "__link_fields__", {})) + literals: dict[str, list[Any]] = dict(getattr(cls, "__link_literals__", {})) + for name, field in cls.model_fields.items(): + extra = field.json_schema_extra + extra = extra if isinstance(extra, dict) else {} + explicit_range = extra.get("x-oold-range") or extra.get("range") + flagged = bool(extra.get("x-oold-link")) + target, many, marked, lits = _unwrap(field.annotation) + if not (explicit_range or flagged or marked): + continue + if explicit_range and not isinstance(target, type): + target = explicit_range + # carry the same promises the binding computes, so Link[T] is + # mandatory here too and x-oold-required-iri is enforced + _, _, optional = _extract_target(field.annotation) + descr = _AutoLink( + name, + target, + many=many, + optional=optional, + required_iri=bool(extra.get("x-oold-required-iri")), + ) + setattr(cls, name, descr) + links[name] = descr + if lits: + literals[name] = lits + cls.__link_fields__ = links + cls.__link_literals__ = literals + # Register the way the binding does - including the inherited-IRI + # guard - rather than writing the type default straight in, which let a + # subclass that only narrows a field replace its parent in the registry. + _register_class(cls) + + def _set_link(self, name: str, value: Any) -> None: + # union arms: a bare string stays a literal when the field also + # declares a literal arm; a reference then arrives as {"@id": ...} + arms = type(self).__link_literals__.get(name) + if arms and isinstance(value, str): + object.__setattr__(self, name, value) + self._links.pop(name, None) + return + type(self).__link_fields__[name].set_value(self, self._coerce(value)) + + @staticmethod + def _coerce(value: Any) -> Any: + def one(v: Any) -> Any: + if isinstance(v, dict) and set(v) == {"@id"}: + return v["@id"] # pure reference object + return v + + if isinstance(value, list): + return [one(v) for v in value] + return one(value) + + def __setattr__(self, name: str, value: Any) -> None: + descr = type(self).__link_fields__.get(name) + if descr is not None: + arms = type(self).__link_literals__.get(name) + if arms and isinstance(value, str): + object.__setattr__(self, name, value) + self._links.pop(name, None) + return + descr.set_value(self, self._coerce(value)) + else: + super().__setattr__(name, value) + + @model_serializer(mode="wrap") + def _serialize_links(self, handler: Any) -> dict[str, Any]: + d = handler(self) + literals = type(self).__link_literals__ + for name in type(self).__link_fields__: + stored = self._links.get(name) + if stored is None and name not in self._links: + # Never set as a link. A literal arm may have taken the value, + # in which case the plain field already serialised it; keep it. + # Otherwise the field is unset and contributes nothing - drop + # the [] the descriptor hands back so it stays out of payloads. + # handler() has already read the descriptor, which caches its + # [] into __dict__, so test the emitted value rather than the + # instance: a literal arm leaves a real value here. + if d.get(name) in (None, [], {}): + d.pop(name, None) + continue + # A field that also accepts a literal cannot emit a reference as a + # bare IRI: on re-read the string would be indistinguishable from + # text. JSON-LD spells the unambiguous form {"@id": ...}. + boxed = bool(literals.get(name)) + emitted = _emit(stored, boxed) + # An explicit empty list is kept - it round-trips as [] and is not + # the same statement as "unset", which is dropped above. + if emitted is None: + d.pop(name, None) + else: + d[name] = emitted + return d diff --git a/src/oold/model/_ref.py b/src/oold/model/_ref.py new file mode 100644 index 0000000..3953364 --- /dev/null +++ b/src/oold/model/_ref.py @@ -0,0 +1,190 @@ +"""Typed reference values for the graph-object binding. + +A :class:`Ref` holds either an unresolved IRI or a resolved object. The +descriptor binding stores links as ``Ref`` instances, which is what lets a +link be inspected (``get_iri_ref``) without triggering resolution, resolved +in batches, and serialised back to an IRI. +""" + +from __future__ import annotations + +from typing import ( + Any, + Generic, + TypeVar, + get_args, +) + +from pydantic import BaseModel, GetCoreSchemaHandler +from pydantic_core import core_schema + +from oold.backend.interface import GetResolverParam, get_resolver + +T = TypeVar("T") + + +class OoldModel(BaseModel): + """Minimal experimental base: JSON round-trip + IRI identity, no metaclass. + + Range fields are declared with the :class:`Ref` type, e.g. + ``b: Optional[Ref[Bar]] = None``. Everything else is plain pydantic. + """ + + def get_iri(self) -> str | None: + """Return the object's IRI (defaults to its ``id`` field).""" + return getattr(self, "id", None) + + def to_json(self, exclude_none: bool = True) -> dict: + """Serialise to a plain dict; :class:`Ref` fields collapse to IRIs.""" + return self.model_dump(exclude_none=exclude_none) + + @classmethod + def ld_context(cls) -> dict: + """JSON-LD context for :meth:`to_jsonld`. Override per model. + + Reference fields must map to ``{"@type": "@id"}`` so that a JSON-LD + processor treats the serialised IRI string as a node reference rather + than a literal. + """ + return {"id": "@id", "type": "@type"} + + def to_jsonld(self) -> dict: + """Return a compact JSON-LD document; ``Ref`` fields are IRI nodes.""" + return {"@context": self.ld_context(), **self.to_json()} + + @classmethod + def from_dict(cls, d: dict) -> OoldModel: + """Construct from a stored dict, ignoring non-field keys (@context...).""" + fields = getattr(cls, "model_fields", {}) + return cls(**{k: v for k, v in d.items() if k in fields}) + + +def _construct(target: type | None, d: Any) -> Any: + if target is None: + return d + if hasattr(target, "from_dict"): + return target.from_dict(d) + return target(**d) + + +def _ref_core_schema(target: type | None) -> core_schema.CoreSchema: + """Pydantic v2 core schema shared by ``Ref[T]`` and ``OoldRange``. + + Validation coerces an IRI string / dict / model / existing ``Ref`` into a + ``Ref``; serialisation emits the IRI. The schema fully replaces the target's + own schema, so the runtime value is a ``Ref`` even when the *declared* type + is the target (the transparent ``Linked[T]`` form). + """ + + def validate(value: Any) -> Ref | None: + if value is None: + return None + if isinstance(value, Ref): + if value._target is None: + value._target = target + return value + if isinstance(value, str): + return Ref(iri=value, target=target) + if isinstance(value, BaseModel): + return Ref(obj=value, target=target) + if isinstance(value, dict): + return Ref(obj=_construct(target, value), target=target) + raise ValueError(f"Cannot coerce {value!r} into Ref[{target}]") + + def serialize(ref: Ref | None) -> str | None: + return None if ref is None else ref.iri + + return core_schema.no_info_plain_validator_function( + validate, + serialization=core_schema.plain_serializer_function_ser_schema(serialize, when_used="always"), + ) + + +class Ref(Generic[T]): + """A typed reference to a linked object, by IRI or by value. + + A ``Ref`` holds either an unresolved ``iri`` or a resolved ``_obj`` (or + both once resolved). It resolves lazily and explicitly: + + - ``ref.resolve()`` returns the target object, fetching it through the + registered backend on first use and caching it, + - attribute access delegates transparently (``foo.b.name`` resolves ``b`` + then reads ``name``) - but, unlike the shipped model, the magic lives on + the reference object, not on every model attribute access. + + Serialisation always emits the IRI, so an object graph round-trips to + IRI-linked JSON / JSON-LD. + """ + + __slots__ = ("_obj", "_target", "iri") + + def __init__( + self, + iri: str | None = None, + obj: T | None = None, + target: type | None = None, + ): + self._obj = obj + self._target = target + if iri is None and obj is not None and hasattr(obj, "get_iri"): + iri = obj.get_iri() + self.iri = iri + + # resolution + + @property + def resolved(self) -> bool: + return self._obj is not None + + def resolve(self) -> T | None: + """Return the target object, resolving via the backend on first use.""" + if self._obj is None and self.iri is not None: + resolver = get_resolver(GetResolverParam(iri=self.iri)).resolver + fetched = resolver.resolve_iris([self.iri]).get(self.iri) + if fetched is None: + raise KeyError(f"Could not resolve reference {self.iri!r}") + self._obj = _construct(self._target, fetched) + return self._obj + + async def aresolve(self) -> T | None: + """Async resolution hook. + + The shipped binding cannot express this at all - resolution is buried + inside synchronous ``__getattribute__``. Here it is an ordinary method, + so an async backend can be awaited. The in-repo backends are sync, so + this simply defers to :meth:`resolve`. + """ + return self.resolve() + + def __getattr__(self, name: str) -> Any: + # Only called for names not found normally (Ref uses __slots__), so it + # never shadows iri/_obj/resolve. Transparent, explicit delegation. + if name.startswith("__") and name.endswith("__"): + # Never resolve for a dunder probe. copy.deepcopy asks for + # __deepcopy__, pickle for __reduce_ex__, and answering those by + # resolving hands back the *target*, so a deep copy replaces every + # Ref with a copy of the object it points at - after which the + # stored references are gone and link_iris/to_json raise. The same + # applies to any hasattr() probe, which would otherwise perform I/O. + raise AttributeError(name) + obj = self.resolve() + return getattr(obj, name) + + def __eq__(self, other: Any) -> bool: + if isinstance(other, Ref): + return self.iri == other.iri + return NotImplemented + + def __hash__(self) -> int: + return hash(self.iri) + + def __repr__(self) -> str: + return f"Ref(iri={self.iri!r}, resolved={self.resolved})" + + # pydantic v2 integration + + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + args = get_args(source_type) + target = args[0] if args else None + return _ref_core_schema(target) diff --git a/src/oold/model/v1/__init__.py b/src/oold/model/v1/__init__.py index 0375536..467ebab 100644 --- a/src/oold/model/v1/__init__.py +++ b/src/oold/model/v1/__init__.py @@ -1,5 +1,6 @@ import json import logging +import os from collections.abc import Callable from typing import ( TYPE_CHECKING, @@ -111,13 +112,13 @@ def __getattr__(self, name): _controller_types: dict[str, list] = {} -M = TypeVar("M", bound="LinkedBaseModel") +M = TypeVar("M", bound="_LinkedBaseModelLegacy") _logger = logging.getLogger(__name__) # pydantic v1 -class LinkedBaseModelMetaClass(pydantic.v1.main.ModelMetaclass): +class _LinkedBaseModelMetaClassLegacy(pydantic.v1.main.ModelMetaclass): _constructing: bool = False """Guards against __getattribute__ intercepting field access during class construction. Pydantic checks ``getattr(base, field_name, None)`` in its @@ -126,11 +127,11 @@ class LinkedBaseModelMetaClass(pydantic.v1.main.ModelMetaclass): of the default None, causing false-positive field-name collision errors.""" def __new__(mcs, name, bases, namespace): - LinkedBaseModelMetaClass._constructing = True + _LinkedBaseModelMetaClassLegacy._constructing = True try: cls = super().__new__(mcs, name, bases, namespace) finally: - LinkedBaseModelMetaClass._constructing = False + _LinkedBaseModelMetaClassLegacy._constructing = False # Register type IRI mapping. Controllers go to _controller_types. _is_ctrl = any(b.__module__ == "oold.model" and b.__name__ == "BaseController" for b in cls.__mro__) @@ -293,12 +294,18 @@ class _LinkedBaseModel(BaseModel, GenericLinkedBaseModel): else: - class _LinkedBaseModel(BaseModel, GenericLinkedBaseModel, metaclass=LinkedBaseModelMetaClass): + class _LinkedBaseModel(BaseModel, GenericLinkedBaseModel, metaclass=_LinkedBaseModelMetaClassLegacy): pass -class LinkedBaseModel(_LinkedBaseModel): - """LinkedBaseModel for pydantic v1""" +class _LinkedBaseModelLegacy(_LinkedBaseModel): + """The per-attribute-interception binding for pydantic v1. + + Exported as ``LinkedBaseModel`` unless ``OOLD_DESCRIPTOR_BINDING=0`` selects + it, which is decided at the bottom of this module. It carries its own name + so that the exported one has a single declaration a type checker can + resolve - pyright keeps the first of two and would ignore the swap. + """ __iris__: dict[str, str | list[str]] | None = PrivateAttr() @@ -336,7 +343,7 @@ def get_iri(self) -> str: return self.id @classmethod - def parse_obj(cls, obj: Any) -> "LinkedBaseModel": + def parse_obj(cls, obj: Any) -> "_LinkedBaseModelLegacy": """Parse the object and return a LinkedBaseModel instance. This method is called by pydantic when creating a new (default) instance of the model.""" @@ -593,7 +600,7 @@ def get_raw(self, field_name: str): @staticmethod def _resolve(iris): resolver = get_resolver(GetResolverParam(iri=iris[0])).resolver - node_dict = resolver.resolve(ResolveParam(iris=iris, model_cls=LinkedBaseModel)).nodes + node_dict = resolver.resolve(ResolveParam(iris=iris, model_cls=_LinkedBaseModelLegacy)).nodes return node_dict def _store(self): @@ -660,10 +667,10 @@ def _recursive_object_to_iri(d: dict, model_obj): for item, model_item in zip(value, model_value, strict=False): if isinstance(item, dict) and hasattr(model_item, "__iris__"): model_item._object_to_iri(item) - LinkedBaseModel._recursive_object_to_iri(item, model_item) + _LinkedBaseModelLegacy._recursive_object_to_iri(item, model_item) elif isinstance(value, dict) and hasattr(model_value, "__iris__"): model_value._object_to_iri(value) - LinkedBaseModel._recursive_object_to_iri(value, model_value) + _LinkedBaseModelLegacy._recursive_object_to_iri(value, model_value) def _raw_dict(self): """Serialize to dict without _object_to_iri at any level. @@ -828,9 +835,9 @@ def to_jsonld(self) -> builtins.dict: return export_jsonld(self, BaseModel) @classmethod - def from_jsonld(cls, jsonld: builtins.dict) -> "LinkedBaseModel": + def from_jsonld(cls, jsonld: builtins.dict) -> "_LinkedBaseModelLegacy": """Constructs a model instance from a JSON-LD representation.""" - return import_jsonld(BaseModel, LinkedBaseModel, cls, jsonld, _types) + return import_jsonld(BaseModel, _LinkedBaseModelLegacy, cls, jsonld, _types) def to_json(self, exclude_defaults: bool = False) -> builtins.dict: """Return the JSON representation of the object as dict. @@ -860,10 +867,32 @@ def to_json(self, exclude_defaults: bool = False) -> builtins.dict: return result @classmethod - def from_json(cls, json_dict: builtins.dict) -> "LinkedBaseModel": + def from_json(cls, json_dict: builtins.dict) -> "_LinkedBaseModelLegacy": """Constructs a model instance from a JSON representation.""" - return import_json(BaseModel, LinkedBaseModel, cls, json_dict, _types) + return import_json(BaseModel, _LinkedBaseModelLegacy, cls, json_dict, _types) # Re-export BaseController from v2 module (it's a plain class, no Pydantic dep) from oold.model import BaseController # noqa: E402, F401 + +# Opt-in descriptor binding, mirroring oold.model. The generated packages emit +# a v1 variant and the production entity models are v1, so the switch has to +# cover this module too or it never exercises the path that matters. +# The branch is taken at import time, which a type checker cannot follow: left +# to infer, it keeps the legacy types for every consumer of this module. Stating +# the default under TYPE_CHECKING is what carries the typed subscript and the +# link annotations over - the same reason as in `oold.model`. +if TYPE_CHECKING: + from oold.model.v1._descriptor import LinkedBaseModel as LinkedBaseModel + from oold.model.v1._descriptor import ( + LinkedBaseModelMetaClass as LinkedBaseModelMetaClass, + ) +elif os.environ.get("OOLD_DESCRIPTOR_BINDING", "1") != "0": + from oold.model.v1 import _descriptor as _descriptor_module + + _descriptor_module.use_type_registry(_types, _controller_types) + LinkedBaseModel = _descriptor_module.LinkedBaseModel + LinkedBaseModelMetaClass = _descriptor_module.LinkedBaseModelMetaClass +else: + LinkedBaseModel = _LinkedBaseModelLegacy + LinkedBaseModelMetaClass = _LinkedBaseModelMetaClassLegacy diff --git a/src/oold/model/v1/_descriptor.py b/src/oold/model/v1/_descriptor.py new file mode 100644 index 0000000..ea8d52f --- /dev/null +++ b/src/oold/model/v1/_descriptor.py @@ -0,0 +1,368 @@ +"""Descriptor binding for pydantic **v1** models. + +The generated packages emit both a v1 and a v2 variant, and the production entity +models are v1, declaring links with the bare keyword form:: + + links: Optional[List[T]] = Field(None, range="T") + +so the v1 path is not optional. Detection is simpler here than in v2: pydantic v1 +already resolves the target into ``field.type_`` and reports list-ness through +``field.shape``, and the extras land in ``field.field_info.extra``. + +The mechanics match the v2 module (:mod:`oold.model._descriptor`): +a **non-data** descriptor per link field, resolved values cached in the instance +``__dict__`` so warm reads never re-enter Python, batched resolution, and the +downstream API surface (``get_iri_ref``, ``__iris__``, ``to_json`` ...) preserved. +""" + +from __future__ import annotations + +import json +from typing import Any, TypeVar, overload + +from pydantic.v1 import BaseModel, PrivateAttr +from pydantic.v1.fields import SHAPE_LIST, SHAPE_SET, SHAPE_TUPLE +from pydantic.v1.main import ModelMetaclass + +from oold.model._compat import LinkedApiMixin +from oold.model._descriptor import ( + Condition, + FieldProxy, + LinkResultList, + _AutoLink, + _Constructing, + _default_iris, +) + +_MANY_SHAPES = {SHAPE_LIST, SHAPE_SET, SHAPE_TUPLE} + +_M = TypeVar("_M") + +_TYPE_REGISTRY: dict[str, type] = {} +"""Type IRI -> model class. + +Kept separate from the v2 registry: the two live in different pydantic worlds, +and a v2 class handed to v1 deserialisation would fail to validate. +""" + +_CONTROLLER_REGISTRY: dict[str, list] = {} + + +def use_type_registry(registry: dict, controllers: dict | None = None) -> None: + """Write registrations into ``registry`` instead of the module-local one. + + Downstream code imports ``oold.model.v1._types`` and writes to it directly, + so the binding has to share that very mapping rather than keep its own - + otherwise resolution silently falls back to the declared target. + """ + global _TYPE_REGISTRY, _CONTROLLER_REGISTRY + registry.update(_TYPE_REGISTRY) + _TYPE_REGISTRY = registry + if controllers is not None: + controllers.update(_CONTROLLER_REGISTRY) + _CONTROLLER_REGISTRY = controllers + + +def _register_class_v1(cls: type) -> None: + """Register a class under the type IRIs it introduces. + + Mirrors the shipped v1 metaclass: controllers are collected separately so + they never shadow the data model they extend, and a class may only claim the + IRIs it introduces itself - a subclass that merely narrows a field reports + its parent's IRI and would otherwise replace it. + """ + from oold.model import _inherited_cls_iris + + iri = cls.get_cls_iri() if hasattr(cls, "get_cls_iri") else None + if iri is None: + return + is_ctrl = any(b.__name__ == "BaseController" for b in cls.__mro__) + inherited = frozenset() if is_ctrl else _inherited_cls_iris(cls) + for value in iri if isinstance(iri, list) else [iri]: + if not isinstance(value, str): + continue + if is_ctrl: + _CONTROLLER_REGISTRY.setdefault(value, []).append(cls) + elif value not in inherited: + _TYPE_REGISTRY[value] = cls + + +def _neutralise_field(field: Any) -> Any: + """Make a link field optional and defaultless at the pydantic level. + + Link values are routed around pydantic - the descriptor holds them - so the + field is always absent from the payload pydantic validates, and a default + left in place would be evaluated on every construction. Generated models + spell a default IRI as ``T.parse_obj("")``, which resolves through the + backend, so leaving it would also turn every construction into a + synchronous fetch. + + Returns the declared default IRI(s) so the caller can hand them to the + descriptor: dropping them outright lost the declared default. + """ + iris = _default_iris(getattr(field, "field_info", None)) or _default_iris(field) + field.required = False + field.allow_none = True + field.default = None + field.default_factory = None + info = getattr(field, "field_info", None) + if info is not None: + info.default = None + info.default_factory = None + return iris + + +class _AutoLinkV1(_AutoLink): + """The shared link descriptor, with v1's way of naming the target. + + Everything else - the construction guard, batched resolution, the instance + cache, ``set_value``, ``iris`` and the comparison operators - was a + copy of the v2 descriptor differing only in how the target is reached: + pydantic v1 resolves it eagerly into ``field.type_``, so there is nothing to + look up later. + """ + + def _target_cls(self, owner: Any = None) -> Any: + return self.target + + +class LinkedBaseModelMetaClass(ModelMetaclass): + """Installs link descriptors and provides the class-level query DSL.""" + + def __new__(mcs, name, bases, namespace, **kwargs): + _Constructing.enter() + try: + cls = super().__new__(mcs, name, bases, namespace, **kwargs) + finally: + _Constructing.leave() + links: dict[str, _AutoLinkV1] = {} + defaults: dict[str, Any] = {} + for base in reversed(cls.__mro__): + links.update(getattr(base, "__link_fields__", {}) or {}) + defaults.update(getattr(base, "__link_defaults__", {}) or {}) + for fname, field in getattr(cls, "__fields__", {}).items(): + extra = getattr(field.field_info, "extra", None) or {} + if not (extra.get("x-oold-range") or extra.get("range") or extra.get("x-oold-link")): + continue + # v1 resolves the target for us: type_ is the item type and shape + # tells us whether the field is to-many + # v1 cannot pass a hyphenated keyword to Field(), so downstream + # spells it with underscores - the legacy v1 binding reads only that + # form. Accept both. + required_iri = bool(extra.get("x_oold_required_iri") or extra.get("x-oold-required-iri")) + # keywords, not positions: the shared __init__ takes + # (name, target, many, optional, required_iri), and passing + # required_iri positionally lands it in `optional` - which makes + # every v1 link mandatory and disables required_iri entirely. + descr = _AutoLinkV1( + fname, + field.type_, + many=field.shape in _MANY_SHAPES, + required_iri=required_iri, + ) + setattr(cls, fname, descr) + links[fname] = descr + default_iris = _neutralise_field(field) + if default_iris is not None: + defaults[fname] = default_iris + cls.__link_fields__ = links + cls.__link_defaults__ = defaults + _register_class_v1(cls) + return cls + + def __getattr__(cls, name: str) -> Any: + if _Constructing.is_active(): + raise AttributeError(name) + if name.startswith("_"): + raise AttributeError(name) + for klass in cls.__mro__: + fields = klass.__dict__.get("__fields__") + if fields and name in fields: + return FieldProxy(name, getattr(fields[name], "default", None)) + raise AttributeError(name) + + @overload + def __getitem__(cls: type[_M], item: str) -> _M | None: ... + + @overload + def __getitem__(cls: type[_M], item: Condition | bool) -> LinkResultList[_M] | None: ... + + @overload + def __getitem__(cls: type[_M], item: list[str]) -> LinkResultList[_M] | None: ... + + def __getitem__(cls, item: Any) -> Any: + return cls.oold_query(item) + + +class LinkedBaseModel(BaseModel, LinkedApiMixin, metaclass=LinkedBaseModelMetaClass): + """pydantic v1 base with the descriptor binding and the downstream API.""" + + _links: dict = PrivateAttr(default_factory=dict) + __link_fields__: dict = {} + __link_defaults__: dict = {} + + class Config: + arbitrary_types_allowed = True + + @classmethod + def oold_query(cls, item: Any) -> Any: + """Resolve ``Model[...]`` against every registered resolver.""" + from oold.backend import interface + from oold.backend.interface import QueryParam, ResolveParam + + node_list: list = [] + for resolver in interface._resolvers.values(): + try: + if isinstance(item, (str, list)): + nodes = resolver.resolve( + ResolveParam( + iris=[item] if isinstance(item, str) else item, + model_cls=cls, + ) + ).nodes.values() + else: + nodes = resolver.query(QueryParam(query=item, model_cls=cls)).nodes.values() + node_list.extend(nodes) + except NotImplementedError: + continue + if isinstance(item, str): + return node_list[0] if node_list else None + return LinkResultList(node_list) if node_list else None + + def __init__(self, *args: Any, **data: Any) -> None: + if args and isinstance(args[0], BaseModel): + source = args[0] + base = source._raw_dict() if hasattr(source, "_raw_dict") else source.dict() + base.pop("type", None) + data = {**{k: v for k, v in base.items() if v is not None}, **data} + link_fields = type(self).__link_fields__ + link_data = {k: data.pop(k) for k in list(data) if k in link_fields} + super().__init__(**data) + # Pydantic writes each field's default into __dict__, and an entry there + # shadows a non-data descriptor - so an unset link would keep returning + # that default (None) and never reach __get__. Dropping the entries hands + # unset links back to the descriptor, which answers [] for to-many and + # None for to-one. + for _name in link_fields: + self.__dict__.pop(_name, None) + # seed the declared default IRI, which the neutralisation above took off + # the field: the link then resolves lazily, like any other + for _name, _iris in type(self).__link_defaults__.items(): + if _name in link_fields and _name not in link_data: + link_data[_name] = _iris + for key, value in link_data.items(): + link_fields[key].set_value(self, value) + missing = [name for name, d in link_fields.items() if d.required_iri and not self._links.get(name)] + if missing: + # see the v2 note: enforced on a true value, not on key presence + raise ValueError(f"{', '.join(sorted(missing))} is required but not set") + + def __setattr__(self, name: str, value: Any, internal: bool = False) -> None: + # internal=True means "write the value as given": BaseController passes + # it through to bypass link handling for controller-only state. + if name == "__iris__": + # delegate to the shared property, so a v1 model gets the same + # replace semantics as a v2 one + LinkedApiMixin.__iris__.fset(self, value) + return + if internal: + super().__setattr__(name, value) + return + descr = type(self).__link_fields__.get(name) + if descr is not None: + descr.set_value(self, value) + else: + super().__setattr__(name, value) + + # -- downstream API ----------------------------------------------------- + + @property + def __iris__(self) -> dict[str, Any]: + out: dict[str, Any] = {} + for name, descr in type(self).__link_fields__.items(): + iris = descr.iris(self) + if iris: + out[name] = iris + return out + + @classmethod + def _fields(cls) -> dict: + return cls.__fields__ + + def _dump(self, **kwargs: Any) -> dict: + return self.dict(**kwargs) + + @classmethod + def get_type_field(cls) -> str: + return "type" + + @classmethod + def get_cls_iri(cls) -> Any: + """The class IRI(s), from ``Config.schema_extra`` and the type default.""" + schema = getattr(getattr(cls, "__config__", None), "schema_extra", None) or {} + if callable(schema): + schema = {} + out: list[str] = [] + for key in ("$id", "x-oold-iri", "iri"): + if key in schema: + out.append(schema[key]) + break + type_field = cls.__fields__.get(cls.get_type_field()) + if type_field is not None: + default = type_field.default + for value in default if isinstance(default, list) else [default]: + if isinstance(value, str) and value not in out: + out.append(value) + if not out: + return None + return out[0] if len(out) == 1 else out + + def dict(self, **kwargs: Any) -> dict[str, Any]: + """v1 serialisation; link fields collapse to their IRIs.""" + exclude_none = kwargs.pop("exclude_none", False) + links = type(self).__link_fields__ + # Reading a link caches the resolved value in __dict__, which pydantic v1 + # serialises - so whether a link had been read changed the output. Drop + # the cache entries for the duration, then restore them. + cached = {name: self.__dict__.pop(name) for name in links if name in self.__dict__} + try: + d = super().dict(**kwargs) + finally: + self.__dict__.update(cached) + for name, descr in links.items(): + iris = descr.iris(self) + if iris: + d[name] = iris + else: + d[name] = None + if exclude_none: + d = {k: v for k, v in d.items() if v is not None} + return d + + def json(self, **kwargs: Any) -> str: + # dict() leaves UUIDs, datetimes and enums as Python objects, so the + # model's own encoder has to do the conversion - plain json.dumps + # rejects them. + encoder = kwargs.pop("encoder", None) or self.__json_encoder__ + kwargs.pop("models_as_dict", None) + return json.dumps(self.dict(**kwargs), default=encoder) + + def to_json(self, exclude_defaults: bool = False) -> dict[str, Any]: + return json.loads(self.json(exclude_none=True, exclude_defaults=exclude_defaults)) + + @classmethod + def from_json(cls, data: dict[str, Any]) -> Any: + from oold.static import import_json + + return import_json(BaseModel, LinkedBaseModel, cls, data, _TYPE_REGISTRY) + + def to_jsonld(self) -> dict[str, Any]: + from oold.static import export_jsonld + + return export_jsonld(self, BaseModel) + + @classmethod + def from_jsonld(cls, jsonld: dict[str, Any]) -> Any: + from oold.static import import_jsonld + + return import_jsonld(BaseModel, LinkedBaseModel, cls, jsonld, _TYPE_REGISTRY) diff --git a/tests/conftest.py b/tests/conftest.py index d5f4429..49a57b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,86 @@ -""" -Dummy conftest.py for oold. +"""Shared fixtures. -If you don't know what this is for, just leave it empty. -Read more about conftest.py under: -- https://docs.pytest.org/en/stable/fixture.html -- https://docs.pytest.org/en/stable/writing_plugins.html +Two registries here are process-wide: resolvers (an unregistered prefix falls +back to whatever else is in it) and the type registry (keyed by type IRI, so two +modules using the same ``type`` default shadow each other). Both have broken a +run already, which is why the fixtures below always restore. """ -# import pytest +import importlib.util + +import pytest + +from oold.backend import interface +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver + + +@pytest.fixture +def linked_store(): + """Register a document store for a prefix, and take it out again after. + + Usage:: + + def test_x(linked_store): + store = linked_store("ex", {"ex:1": {"id": "ex:1", "type": "ex:T"}}) + """ + saved = dict(interface._resolvers) + + def _make(prefix: str, docs: dict | None = None) -> SimpleDictDocumentStore: + store = SimpleDictDocumentStore() + if docs: + store.store_json_dicts(docs) + set_resolver(SetResolverParam(iri=prefix, resolver=store)) + return store + + yield _make + interface._resolvers.clear() + interface._resolvers.update(saved) + + +@pytest.fixture(autouse=True) +def _restore_global_registries(): + """Undo what a *test* registers, in both process-wide registries. + + Restoring the resolvers alone was not enough: the type registry is keyed by + type IRI, and with the descriptor binding as the default it is the same dict + as ``oold.model._types``. Two modules declaring a class with the same + ``type`` default therefore overwrite each other, and which one wins depends + on collection order - running the suite in reverse produced three failures. + + Classes registered at *module import* are left alone: they are set up before + this fixture runs, so the snapshot already contains them. + """ + from oold.model import _descriptor, _types + from oold.model.v1 import _descriptor as _descriptor_v1 + + registries = [interface._resolvers, _types, _descriptor._TYPE_REGISTRY, _descriptor_v1._TYPE_REGISTRY] + saved = [dict(r) for r in registries] + yield + for registry, snapshot in zip(registries, saved, strict=True): + registry.clear() + registry.update(snapshot) + + +def pytest_configure(config): + """Register the ``benchmark`` mark so it is not an unknown-mark warning.""" + config.addinivalue_line("markers", "benchmark(**kwargs): pytest-benchmark group settings") + + +if not importlib.util.find_spec("pytest_benchmark"): + + @pytest.fixture + def benchmark(): + """Run the function once when pytest-benchmark is not installed. + + Without it every test taking this fixture *errors out* rather than + failing, and an error is easy to read as environmental. Two real + regressions sat behind those errors until CI - which does have the + plugin - ran them. Locally the timing is worthless, but the assertions + around it are the point. + """ + + def _run(func, *args, **kwargs): + return func(*args, **kwargs) + + return _run diff --git a/tests/test_auto_descriptor_binding.py b/tests/test_auto_descriptor_binding.py new file mode 100644 index 0000000..c6b2f78 --- /dev/null +++ b/tests/test_auto_descriptor_binding.py @@ -0,0 +1,193 @@ +"""Tests for the auto-installed descriptor binding. + +Covers the recommended variant: link descriptors installed from annotations, so +the declaration syntax is unchanged. See docs/design/graph-object-binding.md. +""" + +import subprocess +import sys + +import pytest + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model._descriptor import ( + Link, + LinkedBaseModel, + LinkList, + OoldExtra, + OoldField, +) + +CALLS = [] + + +class CountingStore(SimpleDictDocumentStore): + def resolve_iris(self, iris): + CALLS.append(list(iris)) + return super().resolve_iris(iris) + + +class Org(LinkedBaseModel): + id: str + name: str | None = None + type: str | None = "ex:Org" + + +class Person(LinkedBaseModel): + id: str + name: str | None = None + type: str | None = "ex:Person" + knows: list["Person"] | None = OoldField(default=None, range="Person") + employer = Link(Org) + friends = LinkList["Person"]() + + +class Employee(Person): + type: str | None = "ex:Employee" + + +Person.model_rebuild() + + +@pytest.fixture() +def store(): + CALLS.clear() + s = CountingStore() + s.store_json_dicts({ + "ex:p2": {"id": "ex:p2", "name": "Bob", "type": "ex:Person"}, + "ex:p3": {"id": "ex:p3", "name": "Carol", "type": "ex:Person"}, + "ex:e1": {"id": "ex:e1", "name": "Dave", "type": "ex:Employee"}, + "ex:acme": {"id": "ex:acme", "name": "ACME", "type": "ex:Org"}, + }) + set_resolver(SetResolverParam(iri="ex", resolver=s)) + return s + + +def test_oold_field_without_arguments(store): + """OoldField() with no args: the target is inferred from the annotation.""" + + class Team(LinkedBaseModel): + id: str + type: str | None = "ex:Team" + members: list[Org] | None = OoldField() + + Team.model_rebuild() + t = Team(id="ex:t1", members=["ex:acme"]) + assert isinstance(t.members[0], Org) and t.members[0].name == "ACME" + assert t.model_dump(exclude_none=True)["members"] == ["ex:acme"] + + +def test_implicit_and_explicit_forms_coexist(store): + p = Person(id="ex:p1", name="Alice", knows=["ex:p2"], employer="ex:acme", friends=["ex:p3"]) + assert set(Person.__link_fields__) == {"knows", "employer", "friends"} + assert isinstance(p.knows[0], Person) and p.knows[0].name == "Bob" + assert isinstance(p.employer, Org) and p.employer.name == "ACME" + assert isinstance(p.friends[0], Person) and p.friends[0].name == "Carol" + + +def test_explicit_descriptors_are_not_pydantic_fields(): + assert set(Person.model_fields) == {"id", "name", "type", "knows"} + + +def test_lazy_and_batched(store): + p = Person(id="ex:p1", knows=["ex:p2", "ex:p3"]) + assert p.link_iris("knows") == ["ex:p2", "ex:p3"] + assert CALLS == [] # nothing resolved yet + assert [x.name for x in p.knows] == ["Bob", "Carol"] + assert CALLS == [["ex:p2", "ex:p3"]] # one batched call + + +def test_cached_read_uses_instance_dict(store): + p = Person(id="ex:p1", knows=["ex:p2"]) + _ = p.knows + n = len(CALLS) + _ = p.knows + assert len(CALLS) == n + # the cache lives in the instance __dict__, shadowing the descriptor + assert "knows" in p.__dict__ + + +def test_mutation_invalidates_cache(store): + p = Person(id="ex:p1", knows=["ex:p2"]) + assert p.knows[0].name == "Bob" + p.knows = ["ex:p3"] + assert p.knows[0].name == "Carol" + assert p.link_iris("knows") == ["ex:p3"] + + +def test_polymorphic_resolution(store): + p = Person(id="ex:p1", knows=["ex:e1"]) + assert isinstance(p.knows[0], Employee) # subclass, not the declared target + + +def test_linked_object_is_validated(): + with pytest.raises(Exception): + Person(id="ex:p1", knows=[{"name": "no id"}]) # 'id' is required + + +def test_list_operations(store): + p = Person(id="ex:p1", knows=["ex:p2", "ex:p3"]) + assert p.knows["ex:p3"].name == "Carol" + assert [x.id for x in p.knows[Person.name == "Bob"]] == ["ex:p2"] + assert list(p.knows.name) == ["Bob", "Carol"] + + +def test_serialisation_to_iris(store): + p = Person(id="ex:p1", name="Alice", knows=["ex:p2"], employer="ex:acme") + d = p.model_dump(exclude_none=True) + assert d["knows"] == ["ex:p2"] + assert d["employer"] == "ex:acme" + assert d["name"] == "Alice" + + +def test_query_dsl_builds_conditions(store): + cond = Person.name == "John" + assert cond.field == "name" and cond.value == "John" + assert (Employee.name == "x").field == "name" # inherited field + assert (Person.employer == "ex:acme").field == "employer" # link descriptor + + +def test_query_by_iri_and_condition(store): + """Model[...] resolves against the registered backends.""" + found = Person["ex:p2"] + assert found is not None and found.name == "Bob" + matches = Person[Person.name == "Bob"] + assert matches is not None + assert [m.id for m in matches] == ["ex:p2"] + # nothing matching returns None rather than an empty result + assert Person["ex:does-not-exist"] is None + + +def test_typed_extras_validate(): + with pytest.raises(Exception): + OoldExtra(range="") + extra = OoldExtra(range="Person", required_iri=True) + assert extra["x-oold-range"] == "Person" + assert extra.range == "Person" and extra.required_iri is True + + +def test_extras_reach_the_json_schema(): + prop = Person.model_json_schema()["$defs"]["Person"]["properties"]["knows"] + assert prop["x-oold-range"] == "Person" + + +@pytest.mark.xfail( + reason=( + "The binding now lives inside oold.model, so importing it runs the " + "package's FieldInfo monkeypatch. That monkeypatch exists for the old " + "FieldProxy/metaclass design; removing it is the last step of the swap. " + "Disabling it locally leaves the shipped suite unchanged (25 passed, " + "same 2 pre-existing errors), so this is expected to pass once the old " + "implementation is retired." + ), + strict=False, +) +def test_does_not_monkeypatch_fieldinfo(): + """The binding must not patch pydantic process-wide (goal, not yet met).""" + code = "import pydantic.fields as pf;import oold.model._descriptor;print(pf.FieldInfo.__name__)" + res = subprocess.run( # noqa: S603 + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert res.returncode == 0, res.stderr + assert res.stdout.strip() == "FieldInfo" diff --git a/tests/test_binding_switch.py b/tests/test_binding_switch.py new file mode 100644 index 0000000..4f03ac4 --- /dev/null +++ b/tests/test_binding_switch.py @@ -0,0 +1,141 @@ +"""The opt-in descriptor binding must keep downstream contracts intact. + +``OOLD_DESCRIPTOR_BINDING=1`` swaps ``LinkedBaseModel`` for the descriptor +implementation. Two names have to move with it, because downstream imports them +and depends on their identity (see docs/design/downstream-migration.md): + +* ``LinkedBaseModelMetaClass`` is subclassed downstream, so a derived metaclass + must stay a subclass of whatever ``LinkedBaseModel`` actually uses - otherwise + the import fails outright with a metaclass conflict; +* ``_types`` is written to downstream, so the binding must share that very + mapping instead of keeping its own - otherwise resolution silently falls back + to the declared target. + +Each case runs in a subprocess: the switch is read at import time. +""" + +import subprocess +import sys +import textwrap + +REPRO = textwrap.dedent( + """ + import warnings; warnings.filterwarnings("ignore") + from oold.model import LinkedBaseModel, LinkedBaseModelMetaClass as ModelMetaclass + import oold.model as m + + hook_ran = {} + + # verbatim downstream shape: a custom metaclass subclassing oold's, then a + # model combining it with a LinkedBaseModel subclass + class QuantityValueMetaclass(ModelMetaclass): + def __new__(mcs, name, bases, namespace, **kwargs): + cls = super().__new__(mcs, name, bases, namespace, **kwargs) + hook_ran[name] = True + return cls + + class OswLike(LinkedBaseModel): + id: str + + class QuantityValue(OswLike, metaclass=QuantityValueMetaclass): + pass + + # __module__, not __name__: both bindings are called LinkedBaseModel, so a + # name check cannot tell them apart and passes whichever is selected. + print("BASE", LinkedBaseModel.__module__) + print("HOOK", hook_ran.get("QuantityValue", False)) + print("METACLASS_MATCHES", isinstance(QuantityValue, type(LinkedBaseModel))) + # registered_types() is `return _types`, so comparing the two is a + # tautology. The property that matters is that the binding writes into that + # very mapping rather than keeping its own. + from oold.model import _descriptor as d + print("REGISTRY_IS_TYPES", d._TYPE_REGISTRY is m._types) + """ +) + + +def run(enabled: bool) -> dict: + import os + + env = dict(os.environ) + env["OOLD_DESCRIPTOR_BINDING"] = "1" if enabled else "0" + proc = subprocess.run( # noqa: S603 + [sys.executable, "-c", REPRO], capture_output=True, text=True, env=env + ) + assert proc.returncode == 0, proc.stderr[-2000:] + return dict(line.split(" ", 1) for line in proc.stdout.strip().splitlines() if " " in line) + + +def test_default_selects_the_descriptor_binding(): + assert run(enabled=True)["BASE"] == "oold.model._descriptor" + + +def test_opting_out_restores_the_legacy_binding(): + assert run(enabled=False)["BASE"] == "oold.model" + + +def test_downstream_metaclass_subclassing_survives_the_switch(): + """The blocker: swapping only the base class breaks this with a conflict.""" + for enabled in (False, True): + out = run(enabled=enabled) + assert out["METACLASS_MATCHES"] == "True", enabled + assert out["HOOK"] == "True", enabled # the custom hook still runs + + +def test_registry_identity_is_preserved_when_the_binding_is_active(): + """Downstream writes into oold.model._types, so the binding must share it. + + Only meaningful with the binding enabled: with it off the descriptor module + is unused and keeps its own mapping, which is harmless. The old form of this + test asserted `registered_types() is _types` for both, which is `return + _types` compared against itself - true whatever the binding does. + """ + assert run(enabled=True)["REGISTRY_IS_TYPES"] == "True" + + +PLAIN = textwrap.dedent( + """ + import warnings; warnings.filterwarnings("ignore") + from pydantic import Field + from oold.model import LinkedBaseModel + + class T(LinkedBaseModel): + id: str + label: str | None = None + + class M(LinkedBaseModel): + id: str + links: list[T] | None = Field(None, json_schema_extra={"range": "T"}) + + M.model_rebuild() + m = M(id="ex:m", links=[T(id="ex:t1", label="one")]) + print("DUMP", m.model_dump(exclude_none=True)["links"]) + print("DESCRIPTORS", bool(getattr(M, "__link_fields__", {}))) + """ +) + + +def run_plain(links: str) -> dict: + import os + + env = dict(os.environ) + env["OOLD_DESCRIPTOR_BINDING"] = "1" + env["OOLD_LINKS"] = links + proc = subprocess.run( # noqa: S603 + [sys.executable, "-c", PLAIN], capture_output=True, text=True, env=env + ) + assert proc.returncode == 0, proc.stderr[-2000:] + return dict(line.split(" ", 1) for line in proc.stdout.strip().splitlines() if " " in line) + + +def test_links_on_collapse_to_iris(): + out = run_plain("1") + assert out["DUMP"] == "['ex:t1']" + assert out["DESCRIPTORS"] == "True" + + +def test_links_off_is_plain_pydantic(): + """OOLD_LINKS=0 turns a model back into plain pydantic, unedited.""" + out = run_plain("0") + assert out["DUMP"] == "[{'id': 'ex:t1', 'label': 'one'}]" # nested, not an IRI + assert out["DESCRIPTORS"] == "False" # nothing installed diff --git a/tests/test_compat_parity.py b/tests/test_compat_parity.py new file mode 100644 index 0000000..4d59558 --- /dev/null +++ b/tests/test_compat_parity.py @@ -0,0 +1,301 @@ +"""Parity between the shipped LinkedBaseModel and the descriptor binding. + +The descriptor binding is only adoptable if downstream keeps working unchanged. +Downstream inherits its API from ``LinkedBaseModel`` via ``OswBaseModel``; the +members asserted here are the ones a scan of the generated ``opensemantic.*`` +packages and the applications built on them found in active use. See +``docs/design/downstream-migration.md``. + +Each behaviour is exercised on the *same* generated-style model declared on both +bases, and the results compared. +""" + +import pytest +from pydantic import Field + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model import _LinkedBaseModelLegacy as LegacyLinkedBaseModel +from oold.model._descriptor import LinkedBaseModel + + +def build(base, tag): + """A model declared exactly the way the code generator emits it.""" + + class T(base): + id: str + label: str | None = None + type: str | None = f"ex:{tag}T" + + class M(base): + id: str + title: str | None = None + type: str | None = f"ex:{tag}M" + links: list[T] | None = Field(None, json_schema_extra={"range": "T"}) + one: T | None = Field(None, json_schema_extra={"range": "T"}) + + return T, M + + +@pytest.fixture(scope="module", autouse=True) +def store(): + s = SimpleDictDocumentStore() + for tag in ("S", "A"): + s.store_json_dicts({ + f"ex:{tag}1": {"id": f"ex:{tag}1", "label": "one", "type": f"ex:{tag}T"}, + f"ex:{tag}2": {"id": f"ex:{tag}2", "label": "two", "type": f"ex:{tag}T"}, + }) + set_resolver(SetResolverParam(iri="ex", resolver=s)) + return s + + +def both(): + """Yield (tag, T, M) for the shipped and the descriptor binding.""" + return [(tag, *build(base, tag)) for base, tag in ((LegacyLinkedBaseModel, "S"), (LinkedBaseModel, "A"))] + + +def normalised(value, tag): + """Strip the per-binding tag so results can be compared literally.""" + return str(value).replace(tag, "#") + + +def collect(fn): + """Run fn against both bindings and return the tag-normalised results.""" + out = [] + for tag, T, M in both(): + out.append(normalised(fn(tag, T, M), tag)) + return out + + +def test_get_iri_ref_shapes_match(): + def probe(tag, T, M): + m = M(id="ex:m", title="x", links=[f"ex:{tag}1", f"ex:{tag}2"], one=f"ex:{tag}1") + return ( + m.get_iri_ref("links"), # list of IRIs + m.get_iri_ref("one"), # single IRI + m.get_iri_ref("title"), # not a link -> None + ) + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_iris_read_matches(): + def probe(tag, T, M): + m = M(id="ex:m", links=[f"ex:{tag}1"], one=f"ex:{tag}1") + return sorted(m.__iris__), m.__iris__.get("one") + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_iris_write_is_honoured(): + """Pattern C: downstream assigns __iris__ directly to fabricate a link.""" + + def probe(tag, T, M): + m = M(id="ex:m2") + m.__iris__ = {"one": f"ex:{tag}1"} + return m.get_iri_ref("one"), type(m.one).__name__, m.one.label + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_to_json_matches(): + def probe(tag, T, M): + m = M(id="ex:m", title="x", links=[f"ex:{tag}1", f"ex:{tag}2"], one=f"ex:{tag}1") + return m.to_json() + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_links_resolve_to_real_objects(): + def probe(tag, T, M): + m = M(id="ex:m", links=[f"ex:{tag}1", f"ex:{tag}2"]) + return [x.label for x in m.links], isinstance(m.links[0], T) + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_raw_dict_lists_every_field(): + """cast() is built on _raw_dict, so a missing key silently drops a field.""" + + def probe(tag, T, M): + m = M(id="ex:m", title="x", one=f"ex:{tag}1") + raw = m._raw_dict() + return sorted(raw), raw["one"], raw["links"] + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_cast_preserves_links(): + def probe(tag, T, M): + m = M(id="ex:m", title="x", one=f"ex:{tag}1") + other = M(m, title="y") # construct from another instance + return other.get_iri_ref("one"), other.title + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_api_surface_present(): + """Every member downstream inherits must exist on the new base.""" + required = [ + "get_iri_ref", + "get_raw", + "to_json", + "from_json", + "to_jsonld", + "from_jsonld", + "cast", + "cast_none_to_default", + "export_schema", + "get_cls_iri", + "store_jsonld", + ] + missing = [a for a in required if not hasattr(LinkedBaseModel, a)] + assert missing == [], f"missing downstream API: {missing}" + + +# -- regressions found by review (these paths were not covered) -------------- + + +def test_controller_to_json_keeps_the_data_fields(): + """A controller whose only model base is the binding's own base class. + + Downstream controllers mix BaseController with a concrete model, which hid + this: the descriptor binding adds LinkedApiMixin to the MRO, the data-model + detection accepted it as the data model, and to_json() then intersected the + payload against an empty field set. + """ + from oold.model import BaseController + + dumped = [] + for base, tag in ((LegacyLinkedBaseModel, "SC"), (LinkedBaseModel, "AC")): + + class C(BaseController, base): + id: str + type: str | None = f"ex:{tag}" + note: str | None = "n" + + dumped.append(sorted(C(id=f"ex:{tag.lower()}").to_json())) + assert dumped[0] == dumped[1], dumped + assert "note" in dumped[1] + + +def test_field_proxy_truthiness_and_default_forwarding_match(): + """Downstream writes `if Model.field:` and `Model.field.startswith(...)`.""" + seen = [] + for base, tag in ((LegacyLinkedBaseModel, "SP"), (LinkedBaseModel, "AP")): + + class B(base): + id: str + type: str | None = f"ex:{tag}" + empty: str | None = None + filled: str | None = "default-name" + + seen.append((bool(B.empty), bool(B.filled), B.filled.upper())) + assert seen[0] == seen[1], seen + + +def test_iris_assignment_replaces_rather_than_merges(): + for tag, _T, M in both(): + m = M(id=f"ex:{tag}m", one=f"ex:{tag}1") + assert m.__iris__, tag + m.__iris__ = {} + assert m.__iris__ == {}, tag + + +def test_get_raw_does_not_invent_a_none_element(): + for tag, _T, M in both(): + m = M(id=f"ex:{tag}m", links=[f"ex:{tag}-unresolved"]) + assert m.get_raw("links") is None, tag + + +def test_required_iri_is_enforced(): + for base, tag in ((LegacyLinkedBaseModel, "SR"), (LinkedBaseModel, "AR")): + target, _model = build(base, tag) + + class R(base): + id: str + type: str | None = f"ex:{tag}" + one: target | None = Field(None, json_schema_extra={"range": f"ex:{tag}T", "x-oold-required-iri": True}) + + R.model_rebuild() + with pytest.raises(ValueError, match="required but not set"): + R(id=f"ex:{tag.lower()}") + + +def test_unset_and_empty_links_serialise_the_same_way(): + """`links=[]` is a different statement from unset, and both must survive.""" + unset, empty = [], [] + for tag, _T, M in both(): + unset.append(normalised(M(id=f"ex:{tag}m").model_dump(), tag)) + empty.append(normalised(M(id=f"ex:{tag}m", links=[]).model_dump()["links"], tag)) + assert unset[0] == unset[1], unset + assert empty[0] == empty[1], empty + + +def test_inline_object_without_an_iri_is_not_dropped(): + """cast() is built on _raw_dict, so losing it there loses it everywhere.""" + seen = [] + for base, tag in ((LegacyLinkedBaseModel, "SI"), (LinkedBaseModel, "AI")): + + class T(base): + id: str | None = None # a blank node: inline, never referenced + label: str | None = None + type: str | None = f"ex:{tag}T" + + class M(base): + id: str + type: str | None = f"ex:{tag}M" + one: T | None = Field(None, json_schema_extra={"range": f"ex:{tag}T"}) + + M.model_rebuild() + seen.append(normalised(M(id=f"ex:{tag}m", one=T(label="anon"))._raw_dict()["one"], tag)) + assert seen[0] == seen[1], seen + assert "anon" in str(seen[1]) + + +def test_iris_assignment_keeps_inline_objects_and_foreign_keys(): + """Replacing the side-dict must not destroy values it never held. + + The side-dict held IRIs only, so clearing it never removed an inline object, + and a key that is not a link field was remembered rather than written over + the model field of that name. + """ + inline, foreign = [], [] + for tag, T, M in both(): + a = M(id=f"ex:{tag}m", one=T(id=f"ex:{tag}inline", label="inline")) + a.__iris__ = {"links": [f"ex:{tag}1"]} + one = a.one + inline.append(one.label if one is not None else None) + + b = M(id=f"ex:{tag}m", title="hello") + b.__iris__ = {"title": f"ex:{tag}notalink"} + foreign.append((b.title, normalised(b.get_iri_ref("title"), tag))) + assert inline[0] == inline[1] == "inline" + assert foreign[0] == foreign[1], foreign + + +def test_nested_plain_models_still_serialise(): + """_raw_dict tested for a hook that only oold models have, so a nested + plain BaseModel came back as the object - and cast() is built on this.""" + from pydantic import BaseModel + + class Nested(BaseModel): + n: int + + seen = [] + for base, tag in ((LegacyLinkedBaseModel, "SN"), (LinkedBaseModel, "AN")): + + class M(base): + id: str + type: str | None = f"ex:{tag}" + nested: Nested | None = None + + seen.append(M(id=f"ex:{tag}", nested={"n": 5})._raw_dict()["nested"]) + assert seen[0] == seen[1] == {"n": 5} diff --git a/tests/test_compat_parity_v1.py b/tests/test_compat_parity_v1.py new file mode 100644 index 0000000..d7b7109 --- /dev/null +++ b/tests/test_compat_parity_v1.py @@ -0,0 +1,125 @@ +"""Parity between the shipped v1 LinkedBaseModel and the v1 descriptor binding. + +The generated packages emit a v1 variant and the production entity models are +v1, declaring links with the bare keyword form ``Field(None, range="T")``, so v1 +parity is a release blocker rather than a nice-to-have. Mirrors +``test_compat_parity.py``, which does the same for v2. +""" + +import pytest +from pydantic.v1 import Field + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model.v1 import _LinkedBaseModelLegacy as LegacyLinkedBaseModel +from oold.model.v1._descriptor import LinkedBaseModel as LinkedBaseModelV1 + + +def build(base, tag): + """A model declared the way the v1 code generator emits it.""" + + class T(base): + id: str + label: str | None = None + type: str | None = f"ex:{tag}T" + + class M(base): + id: str + title: str | None = None + type: str | None = f"ex:{tag}M" + links: list[T] | None = Field(None, range="T") + one: T | None = Field(None, range="T") + + return T, M + + +@pytest.fixture(scope="module", autouse=True) +def store(): + s = SimpleDictDocumentStore() + for tag in ("SV", "AV"): + s.store_json_dicts({ + f"ex:{tag}1": {"id": f"ex:{tag}1", "label": "one", "type": f"ex:{tag}T"}, + f"ex:{tag}2": {"id": f"ex:{tag}2", "label": "two", "type": f"ex:{tag}T"}, + }) + set_resolver(SetResolverParam(iri="ex", resolver=s)) + return s + + +def both(): + return [(tag, *build(base, tag)) for base, tag in ((LegacyLinkedBaseModel, "SV"), (LinkedBaseModelV1, "AV"))] + + +def collect(fn): + """Run fn against both bindings, tag-normalised so results compare literally.""" + return [str(fn(tag, T, M)).replace(tag, "#") for tag, T, M in both()] + + +def test_bare_range_kwarg_is_detected(): + def probe(tag, T, M): + m = M(id="ex:m", links=[f"ex:{tag}1"], one=f"ex:{tag}1") + return type(m.links[0]).__name__, m.links[0].label, m.one.label + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_get_iri_ref_shapes_match(): + def probe(tag, T, M): + m = M(id="ex:m", title="x", links=[f"ex:{tag}1", f"ex:{tag}2"], one=f"ex:{tag}1") + return ( + m.get_iri_ref("links"), + m.get_iri_ref("one"), + m.get_iri_ref("title"), + ) + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_iris_read_and_write(): + def probe(tag, T, M): + m = M(id="ex:m", links=[f"ex:{tag}1"]) + read = sorted(m.__iris__) + m2 = M(id="ex:m2") + m2.__iris__ = {"one": f"ex:{tag}2"} + return read, m2.get_iri_ref("one"), m2.one.label + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_dict_and_to_json_match(): + def probe(tag, T, M): + m = M(id="ex:m", title="x", links=[f"ex:{tag}1", f"ex:{tag}2"], one=f"ex:{tag}1") + return m.dict(exclude_none=True), m.to_json() + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_raw_dict_lists_every_field(): + def probe(tag, T, M): + m = M(id="ex:m", title="x", one=f"ex:{tag}1") + raw = m._raw_dict() + return sorted(raw), raw["one"], raw["links"] + + shipped, auto = collect(probe) + assert shipped == auto + + +def test_api_surface_present(): + required = [ + "get_iri_ref", + "get_raw", + "to_json", + "from_json", + "to_jsonld", + "from_jsonld", + "cast", + "cast_none_to_default", + "get_cls_iri", + "dict", + "json", + ] + missing = [a for a in required if not hasattr(LinkedBaseModelV1, a)] + assert missing == [], f"missing downstream API: {missing}" diff --git a/tests/test_downstream_shapes.py b/tests/test_downstream_shapes.py new file mode 100644 index 0000000..421319d --- /dev/null +++ b/tests/test_downstream_shapes.py @@ -0,0 +1,341 @@ +"""Declaration shapes that only appear in generated downstream packages. + +Both are cases the shipped binding tolerates by accident and the descriptor +binding broke on the first real run against a generated package: + +1. a subclass **redeclaring** an inherited link field - pydantic inspects the + bases for a same-named attribute and rejects the field when it finds one, and + the installed descriptor is exactly such an attribute; +2. a link field whose declared default is ``T.parse_obj("")`` - a default + that can only ever raise, and which pydantic evaluates as soon as the link + value is routed out of the payload. +""" + +import contextlib + +import pytest +from pydantic import Field +from pydantic.v1 import BaseModel as BaseModelV1 +from pydantic.v1 import Field as FieldV1 + +from oold.model._descriptor import LinkedBaseModel, OoldField +from oold.model.v1._descriptor import LinkedBaseModel as LinkedBaseModelV1 + + +class Target(LinkedBaseModel): + id: str | None = None + label: str | None = None + + +class TargetV1(BaseModelV1): + id: str | None = None + label: str | None = None + + +def test_subclass_may_redeclare_a_link_field(): + class Base(LinkedBaseModel): + id: str + ref: Target | None = Field(None, json_schema_extra={"range": "Target"}) + + class Derived(Base): + # narrowing or re-annotating an inherited link is what generated + # packages do whenever a subschema restates a property + ref: Target | None = Field(None, json_schema_extra={"range": "Target"}) + + d = Derived(id="ex:d", ref="ex:t") + assert d.link_iris("ref") == "ex:t" + + +def test_subclass_may_redeclare_a_link_field_v1(): + class Base(LinkedBaseModelV1): + id: str + ref: TargetV1 | None = FieldV1(None, range="Target") + + class Derived(Base): + ref: TargetV1 | None = FieldV1(None, range="Target") + + d = Derived(id="ex:d", ref="ex:t") + assert d.link_iris("ref") == "ex:t" + + +def _explode(_cls): + raise ValueError("a model cannot be parsed from an IRI string") + + +def test_link_field_default_is_never_evaluated(): + """The declared default is dead weight - the descriptor owns the value.""" + + class M(LinkedBaseModel): + id: str + ref: Target = Field( + default_factory=lambda: _explode(Target), + json_schema_extra={"range": "Target"}, + ) + + assert M(id="ex:m").link_iris("ref") is None # unset, default not evaluated + assert M(id="ex:m", ref="ex:t").link_iris("ref") == "ex:t" + + +def test_link_field_default_is_never_evaluated_v1(): + class M(LinkedBaseModelV1): + id: str + ref: TargetV1 = FieldV1(default_factory=lambda: _explode(TargetV1), range="Target") + + assert M(id="ex:m").link_iris("ref") is None + assert M(id="ex:m", ref="ex:t").link_iris("ref") == "ex:t" + + +@pytest.mark.parametrize("base", [LinkedBaseModel, LinkedBaseModelV1]) +def test_setattr_accepts_the_internal_flag(base): + """``BaseController.__setattr__`` forwards ``internal=`` to the model.""" + field = Field if base is LinkedBaseModel else FieldV1 + extra = {"json_schema_extra": {"range": "Target"}} if base is LinkedBaseModel else {"range": "Target"} + target = Target if base is LinkedBaseModel else TargetV1 + + class M(base): + id: str + ref: target | None = field(None, **extra) + + m = M(id="ex:m") + # internal=True writes the value as given, bypassing link handling + m.__setattr__("id", "ex:other", internal=True) + assert m.id == "ex:other" + + +def test_v1_to_json_encodes_non_json_types(): + """dict() leaves UUID/datetime as objects; to_json() must not.""" + import json + from datetime import datetime, timezone + from uuid import UUID + + class Doc(LinkedBaseModelV1): + uuid: UUID + at: datetime + ref: TargetV1 | None = FieldV1(None, range="Target") + + doc = Doc( + uuid=UUID("6dd0a5aa-8b53-4b0f-8a1d-2b1b1a1f0c11"), + at=datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + ref="ex:t", + ) + out = doc.to_json() + assert out["uuid"] == "6dd0a5aa-8b53-4b0f-8a1d-2b1b1a1f0c11" + assert out["ref"] == "ex:t" + json.dumps(out) # the whole point: the result is JSON-serialisable + + +def test_unset_links_honour_the_exclude_flags(): + """The unset-key was written after handler(), so it survived every + exclusion - putting an explicit null into every stored document.""" + + class M(LinkedBaseModel): + id: str + one: Target | None = Field(None, json_schema_extra={"range": "Target"}) + links: list[Target] | None = Field(None, json_schema_extra={"range": "Target"}) + + M.model_rebuild() + m = M(id="ex:m", one="ex:1") + assert "links" not in m.to_json() + assert "links" not in m.model_dump(exclude_none=True) + assert "links" not in m.model_dump(exclude={"links"}) + assert m.model_dump()["links"] is None # still there when nothing is excluded + + +def test_deepcopy_keeps_links_as_references(): + """Ref.__getattr__ delegated dunders to resolve(), so copy.deepcopy asked + for __deepcopy__ and got the target back - replacing every Ref with a copy + of the object it pointed at.""" + import copy + + class M(LinkedBaseModel): + id: str + ref: Target | None = Field(None, json_schema_extra={"range": "Target"}) + + m = M(id="ex:m", ref="ex:1") + copied = copy.deepcopy(m) + assert copied.link_iris("ref") == "ex:1" + assert copied.to_json()["ref"] == "ex:1" + + +def test_optional_wrapping_a_link_is_optional(): + """Optional[Link[T]] and Link[T | None] mean the same thing; the union was + only honoured when it came *after* the Link.""" + from typing import Optional + + from oold.model import Link, LinkNotResolved + + class M(LinkedBaseModel): + id: str + outer: Optional[Link[Target]] = OoldField() # noqa: UP045 - the spelling under test + inner: Link["Target | None"] = OoldField() + mandatory: Link[Target] = OoldField() + + M.model_rebuild() + m = M(id="ex:m") + assert m.outer is None + assert m.inner is None + with pytest.raises(LinkNotResolved): + _ = m.mandatory + + +def test_a_shared_field_info_is_not_mutated(): + """Field() objects get reused across models; neutralising in place stripped + the default process-wide, including from plain BaseModels.""" + from pydantic import BaseModel + + shared = Field(default_factory=list, json_schema_extra={"range": "Target"}) + + class Linked(LinkedBaseModel): + id: str + links: list[Target] | None = shared + + class Plain(BaseModel): + links: list[int] | None = shared + + assert Plain().links == [] + assert shared.default_factory is not None + + +def test_annotated_link_default_is_never_evaluated(): + """A Field() in Annotated metadata was not seen, so its default survived.""" + from typing import Annotated + + class M(LinkedBaseModel): + id: str + ref: Annotated[Target, Field(default_factory=lambda: _explode(Target), json_schema_extra={"range": "Target"})] + + assert M(id="ex:m").link_iris("ref") is None + assert M(id="ex:m", ref="ex:t").link_iris("ref") == "ex:t" + + +def test_models_are_unhashable_like_pydantic(): + class M(LinkedBaseModel): + id: str + + with pytest.raises(TypeError): + hash(M(id="ex:m")) + + +def test_extras_take_part_in_equality(): + from pydantic import ConfigDict + + class E(LinkedBaseModel): + model_config = ConfigDict(extra="allow") + id: str + + assert E(id="ex:a", foo=1) != E(id="ex:a", foo=2) + + +def test_a_type_array_registers_under_every_iri(): + from pydantic import ConfigDict + + # the descriptor module's own registry: it is only the same object as + # oold.model._types when the binding is active, and this file always + # exercises the descriptor classes directly + from oold.model._descriptor import _TYPE_REGISTRY + + class TA(LinkedBaseModel): + model_config = ConfigDict(json_schema_extra={"$id": "ex:TAid"}) + type: list[str] | None = ["ex:TA1", "ex:TA2"] + + assert TA.get_cls_iri() == ["ex:TAid", "ex:TA1", "ex:TA2"] + assert all(_TYPE_REGISTRY.get(i) is TA for i in ("ex:TAid", "ex:TA1", "ex:TA2")) + assert TA().type == ["ex:TA1", "ex:TA2"] # serialisation keeps the array + + +def test_v1_dict_does_not_depend_on_whether_a_link_was_read(): + class T1(LinkedBaseModelV1): + id: str + type: str | None = "v1s:T" + + class M1(LinkedBaseModelV1): + id: str + type: str | None = "v1s:M" + links: list[T1] | None = FieldV1(None, range="v1s:T") + + M1.update_forward_refs() + m = M1(id="v1s:m", links=["v1s:unresolvable"]) + before = m.dict() + with contextlib.suppress(Exception): + _ = m.links # caches whatever resolution produced + assert m.dict() == before, "reading a link changed the serialised output" + + +def test_link_fields_honour_aliases_in_both_directions(): + """Every other key honours by_alias, so a link under its field name made a + payload that mixed both spellings - and a by_alias payload could not be read + back, because the alias was left for pydantic to validate against the target.""" + + class M(LinkedBaseModel): + id: str + one: Target | None = Field(None, alias="oneAlias", json_schema_extra={"range": "Target"}) + + m = M(**{"id": "ex:m", "oneAlias": "ex:1"}) + assert m.link_iris("one") == "ex:1" + assert m.model_dump(by_alias=True, exclude_none=True)["oneAlias"] == "ex:1" + assert m.model_dump(exclude_none=True)["one"] == "ex:1" + + +def test_oold_field_accepts_a_default_factory(): + class M(LinkedBaseModel): + id: str + links: list[Target] = OoldField(default_factory=list, range="Target") + + assert M(id="ex:m").link_iris("links") == [] + + +def test_a_declared_default_iri_survives(linked_store): + """The pydantic-level default has to be stripped - it is evaluated on every + construction and generated code spells it as a backend call - but the IRI it + names is the declaration, not dead weight. Dropping it outright lost the + default, and the field read back as None.""" + linked_store("ex", {"ex:dflt": {"id": "ex:dflt", "label": "default target"}}) + + class M(LinkedBaseModel): + id: str + # exactly what datamodel-code-generator emits for `"default": "ex:dflt"` + ref: Target | None = Field( + default_factory=lambda: Target.model_validate("ex:dflt"), + json_schema_extra={"range": "Target"}, + ) + + M.model_rebuild() + m = M(id="ex:m") + assert m.link_iris("ref") == "ex:dflt" # recorded without resolving + assert m.ref is not None and m.ref.label == "default target" + assert m.to_json()["ref"] == "ex:dflt" + # an explicit value still wins over the default + assert M(id="ex:m", ref="ex:other").link_iris("ref") == "ex:other" + + +def test_clearing_a_link_removes_it_from_the_payload(): + """`m.one = None` and "never set" are the same statement. Treating them + differently left an explicit null behind after a caller had cleared it.""" + + class M(LinkedBaseModel): + id: str + one: Target | None = Field(None, json_schema_extra={"range": "Target"}) + + M.model_rebuild() + m = M(id="ex:m", one="ex:1") + m.one = None + assert m.one is None + assert "one" not in m.to_json() + + +def test_serialising_after_a_partial_resolution(linked_store): + """Reading a to-many link caches the result in __dict__, where pydantic's + own serializer finds it. When one entry could not be resolved the cache + holds a None among the objects, and `list[Target]` cannot render it - + to_json() died with "type object 'NoneType' has no attribute + model_fields".""" + linked_store("ex", {"ex:ok": {"id": "ex:ok", "label": "resolvable"}}) + + class M(LinkedBaseModel): + id: str + refs: list[Target] | None = Field(None, json_schema_extra={"range": "Target"}) + + M.model_rebuild() + m = M(id="ex:m", refs=["ex:ok", "ex:missing"]) + assert [r.label if r else None for r in m.refs] == ["resolvable", None] + assert m.to_json()["refs"] == ["ex:ok", "ex:missing"] diff --git a/tests/test_link_annotation.py b/tests/test_link_annotation.py new file mode 100644 index 0000000..eff246b --- /dev/null +++ b/tests/test_link_annotation.py @@ -0,0 +1,290 @@ +"""``Link[T]`` / ``LinkList[T]`` used as the whole annotation. + +The static half of this contract lives in ``tests/typing/links.py``; here is the +runtime half. Two things to hold down: + +* the annotation form must be indistinguishable from the plain spelling - same + schema, same resolution, same serialisation. It only adds what a checker sees; +* optionality is **declared**. ``Link[T]`` promises a ``T``, so the binding keeps + that promise rather than handing back a ``None`` the type denies; + ``Link[T | None]`` says absence is data and returns ``None``. +""" + +import pytest +from pydantic import Field + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model._descriptor import ( + Link, + LinkedBaseModel, + LinkList, + LinkNotResolved, + OoldField, +) + + +class Org(LinkedBaseModel): + id: str + name: str | None = None + type: str | None = "annot:Organization" + + +class Person(LinkedBaseModel): + id: str + name: str | None = None + type: str | None = "annot:Person" + knows: LinkList["Person | None"] = OoldField() + employer: Link["Org | None"] = OoldField() + mixed: LinkList["Person | Org | None"] = OoldField() + + +class Employee(LinkedBaseModel): + """Every employee has an employer - declared, and enforced.""" + + id: str + type: str | None = "annot:Employee" + employer: Link[Org] = OoldField() + + +class Plain(LinkedBaseModel): + id: str + type: str | None = "annot:Plain" + knows: list["Person"] | None = Field(None, json_schema_extra={"range": "Person"}) + + +Person.model_rebuild() +Employee.model_rebuild() +Plain.model_rebuild() + + +@pytest.fixture +def store(): + store = SimpleDictDocumentStore() + store.store_json_dicts({ + "annot:bob": {"id": "annot:bob", "name": "Bob", "type": "annot:Person"}, + "annot:acme": {"id": "annot:acme", "name": "ACME", "type": "annot:Organization"}, + }) + set_resolver(SetResolverParam(iri="annot", resolver=store)) + return store + + +def test_annotation_alone_declares_the_link(): + """No range= and no x-oold-link needed: the annotation says it.""" + assert set(Person.__link_fields__) == {"knows", "employer", "mixed"} + assert Person.__link_fields__["knows"].many is True + assert Person.__link_fields__["employer"].many is False + + +def test_schema_matches_the_plain_spelling(): + """Pydantic is handed the target's schema, so the output is unchanged.""" + + def props(model): + schema = model.model_json_schema() + return schema.get("properties") or schema["$defs"][model.__name__]["properties"] + + annotated = props(Person)["knows"] + plain = props(Plain)["knows"] + assert annotated["type"] == "array" + assert plain["anyOf"][0]["items"] == {"$ref": "#/$defs/Person"} + # the annotated form carries the None arm it declares + assert {"$ref": "#/$defs/Person"} in annotated["items"]["anyOf"] + + +def test_construct_by_iri_object_and_json(store): + p = Person( + id="annot:a", + knows=["annot:bob", Person(id="annot:c", name="Carol"), {"id": "annot:d"}], + employer="annot:acme", + ) + assert [type(v).__name__ for v in p.knows] == ["Person", "Person", "Person"] + assert p.knows[0].name == "Bob" + assert isinstance(p.employer, Org) + assert p.employer.name == "ACME" + + +def test_mixed_target_resolves_each_arm_by_type(store): + p = Person(id="annot:a", mixed=["annot:bob", "annot:acme"]) + assert [type(v).__name__ for v in p.mixed] == ["Person", "Org"] + + +def test_serialises_back_to_iris(store): + p = Person(id="annot:a", knows=["annot:bob"], employer="annot:acme") + dumped = p.model_dump(exclude_none=True) + assert dumped["knows"] == ["annot:bob"] + assert dumped["employer"] == "annot:acme" + + +def test_explicit_descriptor_form_still_works(store): + """The same classes remain usable as unannotated descriptors.""" + + class Explicit(LinkedBaseModel): + id: str + type: str | None = "annot:Explicit" + employer = Link(Org) + knows = LinkList("Person") + + e = Explicit(id="annot:e", employer="annot:acme") + assert isinstance(e.employer, Org) + assert e.employer.name == "ACME" + + +# -- optionality is declared ------------------------------------------------- + + +def test_optional_link_unset_reads_as_none(store): + assert Person(id="annot:a").employer is None + + +def test_optional_link_keeps_the_slot_of_an_unresolvable_reference(store): + """The list stays aligned with the stored references.""" + p = Person(id="annot:a", knows=["annot:bob", "annot:nobody"]) + assert len(p.knows) == 2 + assert p.knows[1] is None + assert p.link_iris("knows") == ["annot:bob", "annot:nobody"] + + +def test_mandatory_link_unset_raises_on_access_not_on_construction(store): + """Partial graph data must still load; the promise is about reading.""" + e = Employee(id="annot:e") # builds fine + with pytest.raises(LinkNotResolved, match="not set"): + _ = e.employer + + +def test_a_whole_chain_needs_one_except_not_a_guard_per_hop(store): + """The point of declaring a link mandatory.""" + + class Node(LinkedBaseModel): + id: str + type: str | None = "annot:Node" + parent: Link["Node"] = OoldField() + + Node.model_rebuild() + leaf = Node(id="annot:leaf", parent={"id": "annot:mid"}) + with pytest.raises(LinkNotResolved): + _ = leaf.parent.parent.parent + + +def test_mandatory_link_resolves(store): + e = Employee(id="annot:e", employer="annot:acme") + assert isinstance(e.employer, Org) + assert e.employer.name == "ACME" + + +def test_mandatory_link_raises_when_the_backend_has_no_such_entity(store): + """Only detectable on access - and None would deny the declared type.""" + e = Employee(id="annot:e", employer="annot:ghost") + with pytest.raises(LinkNotResolved, match="declared mandatory"): + _ = e.employer + + +def test_backend_errors_are_not_mistaken_for_absence(store): + """A transport failure is not 'has no employer' - it propagates.""" + + class Boom(type(store)): + def resolve_iris(self, iris): + raise ConnectionError("backend unreachable") + + set_resolver(SetResolverParam(iri="annot", resolver=Boom())) + e = Employee(id="annot:e", employer="annot:acme") + with pytest.raises(ConnectionError): + _ = e.employer + + +def test_union_target_resolves_on_a_jsonld_backend(): + """The case the broad fallback hid. + + A union target is not a class, so it cannot be a ``model_cls``. Passing it + made ``ResolveParam`` validation fail, which dropped into ``_batch_resolve``'s + ``except Exception`` and rebuilt the *raw* document - fine for a JSON store, + broken for every JSON-LD one, which is where this runs. + """ + from pydantic import ConfigDict + from rdflib import Graph + + from oold.backend.sparql import LocalSparqlBackend + + UN = "https://union.example/" + context = { + "@context": {"id": "@id", "type": "@type", "name": UN + "name"}, + "iri": UN + "Base", + } + + class UBase(LinkedBaseModel): + model_config = ConfigDict(json_schema_extra=context) + id: str + name: str | None = None + + def get_iri(self): + return self.id + + class UPerson(UBase): + model_config = ConfigDict(json_schema_extra={**context, "iri": UN + "Person"}) + type: str | None = UN + "Person" + + class UOrg(UBase): + model_config = ConfigDict(json_schema_extra={**context, "iri": UN + "Org"}) + type: str | None = UN + "Org" + + class UHolder(UBase): + model_config = ConfigDict(json_schema_extra={**context, "iri": UN + "Holder"}) + type: str | None = UN + "Holder" + mixed: LinkList["UPerson | UOrg | None"] = OoldField() + + UHolder.model_rebuild() + + store = LocalSparqlBackend(graph=Graph()) + store.store_jsonld_dicts({ + UN + "bob": UPerson(id=UN + "bob", name="Bob").to_jsonld(), + UN + "acme": UOrg(id=UN + "acme", name="ACME").to_jsonld(), + }) + set_resolver(SetResolverParam(iri="https", resolver=store)) + h = UHolder(id=UN + "h", mixed=[UN + "bob", UN + "acme"]) + assert [type(v).__name__ for v in h.mixed] == ["UPerson", "UOrg"] + assert [v.name for v in h.mixed] == ["Bob", "ACME"] + + +def test_a_malformed_document_reports_its_own_error(store): + """The fallback used to swallow it and mis-construct against the target.""" + + class Broken(type(store)): + def resolve_iris(self, iris): + return {i: {"id": i, "type": "annot:Person", "name": {"not": "a string"}} for i in iris} + + set_resolver(SetResolverParam(iri="annot", resolver=Broken())) + p = Person(id="annot:a", knows=["annot:x"]) + with pytest.raises(Exception) as excinfo: + _ = p.knows + # the model's own validation error, not a downstream TypeError from + # re-constructing an expanded document against the declared target + assert "name" in str(excinfo.value) + + +def test_mutating_a_link_list_never_discards_an_unresolved_reference(store): + """_sync rebuilt storage from the resolved values, so a slot that could not + be resolved was deleted - the list shrank and the IRI was lost.""" + p = Person(id="annot:a", knows=["annot:bob", "annot:nobody"]) + assert p.knows[1] is None + p.knows.append(Person(id="annot:c")) + assert p.link_iris("knows") == ["annot:bob", "annot:nobody", "annot:c"] + + +@pytest.mark.parametrize( + "mutate,expected", + [ + (lambda lst: lst.__setitem__(0, Person(id="annot:z")), ["annot:z", "annot:bob"]), + (lambda lst: lst.pop(), ["annot:acme"]), + (lambda lst: lst.insert(0, Person(id="annot:z")), ["annot:z", "annot:acme", "annot:bob"]), + (lambda lst: lst.clear(), []), + (lambda lst: lst.reverse(), ["annot:bob", "annot:acme"]), + (lambda lst: lst.__delitem__(0), ["annot:bob"]), + (lambda lst: lst.__iadd__([Person(id="annot:z")]), ["annot:acme", "annot:bob", "annot:z"]), + ], +) +def test_every_list_mutation_reaches_storage(store, mutate, expected): + """Only append/remove/extend synced; the rest changed the visible list while + storage kept the old references.""" + p = Person(id="annot:a", mixed=["annot:acme", "annot:bob"]) + values = p.mixed + mutate(values) + assert p.link_iris("mixed") == expected diff --git a/tests/test_notation.py b/tests/test_notation.py new file mode 100644 index 0000000..7784c81 --- /dev/null +++ b/tests/test_notation.py @@ -0,0 +1,296 @@ +"""Tests for the reviewed link declaration notations. + +Type IRIs are prefixed ``ex:N...`` so they do not collide with other test +modules: the class registry used for polymorphic resolution is process-wide +and keyed by type IRI, so two classes claiming the same IRI shadow each other. + +Covers the notations proposed in the oold-python#107 review: +``OoldField()`` / ``link=True`` with the target inferred from the annotation, +``Link[T]`` used *inside* an annotation, and union arms mixing a literal, an +inline object and a reference. +""" + +import pytest +from pydantic import Field + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model import LinkNotResolved +from oold.model._notation import Link, OoldField, OoldModel + + +class Org(OoldModel): + id: str + name: str | None = None + type: str | None = "ex:NOrg" + + +class Location(OoldModel): + id: str | None = None + address: str | None = None + type: str | None = "ex:NLoc" + + +class Person(OoldModel): + id: str + name: str | None = None + type: str | None = "ex:NPerson" + # target inferred from the annotation, no range= needed + knows: list["Person"] = OoldField() + # Link[T] inside the annotation + employer: Link[Org] | None = Field(default=None) + friends: list[Link["Person"]] = OoldField() + # union: literal text | inline object | reference + location: str | Location | None = OoldField(link=True) + + +Person.model_rebuild() + + +@pytest.fixture() +def store(): + s = SimpleDictDocumentStore() + s.store_json_dicts({ + "ex:p2": {"id": "ex:p2", "name": "Bob", "type": "ex:NPerson"}, + "ex:acme": {"id": "ex:acme", "name": "ACME", "type": "ex:NOrg"}, + "ex:loc": { + "id": "ex:loc", + "address": "Champ de Mars", + "type": "ex:NLoc", + }, + }) + set_resolver(SetResolverParam(iri="ex", resolver=s)) + return s + + +def test_all_notations_register_as_links(): + assert set(Person.__link_fields__) == {"knows", "employer", "friends", "location"} + + +def test_oold_field_without_arguments(store): + p = Person(id="ex:p1", knows=["ex:p2"]) + assert isinstance(p.knows[0], Person) + assert p.knows[0].name == "Bob" + assert p.model_dump(exclude_none=True)["knows"] == ["ex:p2"] + + +def test_link_inside_annotation(store): + p = Person(id="ex:p1", employer="ex:acme", friends=["ex:p2"]) + assert isinstance(p.employer, Org) and p.employer.name == "ACME" + assert isinstance(p.friends[0], Person) and p.friends[0].name == "Bob" + d = p.model_dump(exclude_none=True) + assert d["employer"] == "ex:acme" + assert d["friends"] == ["ex:p2"] + + +def test_union_literal_arm(store): + p = Person(id="ex:a", location="at the Eiffel Tower") + assert p.location == "at the Eiffel Tower" + assert p.model_dump(exclude_none=True)["location"] == "at the Eiffel Tower" + + +def test_union_reference_arm(store): + p = Person(id="ex:b", location={"@id": "ex:loc"}) + assert isinstance(p.location, Location) + assert p.location.address == "Champ de Mars" + # the field also accepts a literal, so a reference must be boxed as + # {"@id": ...} - a bare IRI would be re-read as text + assert p.model_dump(exclude_none=True)["location"] == {"@id": "ex:loc"} + + +def test_union_inline_arm_with_id(store): + p = Person( + id="ex:c", + location={"id": "ex:inline", "address": "inline addr", "type": "ex:NLoc"}, + ) + assert isinstance(p.location, Location) and p.location.address == "inline addr" + # it carries an IRI, so it serialises as a (boxed) reference + assert p.model_dump(exclude_none=True)["location"] == {"@id": "ex:inline"} + + +def test_union_inline_without_id_is_a_blank_node(store): + p = Person(id="ex:d", location={"address": "no id here", "type": "ex:NLoc"}) + assert isinstance(p.location, Location) + assert p.link_iris("location") is None + dumped = p.model_dump(exclude_none=True)["location"] + # no IRI to reference, so the object stays nested + assert isinstance(dumped, dict) and dumped["address"] == "no id here" + + +def test_mutation(store): + p = Person(id="ex:p1", knows=["ex:p2"]) + p.knows = [] + assert p.knows == [] + p.knows = ["ex:p2"] + assert p.knows[0].name == "Bob" + + +def test_query_dsl_still_available(store): + """Runs the real query, not a stub. + + ``OoldModel.oold_query`` used to return ``("query", cls.__name__, item)`` + unconditionally, so asserting ``is not None`` here could never fail and no + backend was ever consulted. + """ + cond = Person.name == "Bob" + assert cond.field == "name" + found = Person[cond] + assert found is not None + assert [p.id for p in found] == ["ex:p2"] + assert Person[Person.name == "nobody-by-that-name"] is None + + +def test_union_round_trip_preserves_every_arm(store): + """Deserialisation is the hard part: each arm must survive a round trip.""" + cases = { + "text": ("at the Eiffel Tower", str), + "reference": ({"@id": "ex:loc"}, Location), + "inline": ({"address": "Main St", "type": "ex:NLoc"}, Location), + } + for label, (value, expected) in cases.items(): + original = Person(id="ex:rt", location=value) + restored = Person(**original.model_dump(exclude_none=True)) + assert isinstance(restored.location, expected), label + if expected is str: + assert restored.location == original.location, label + else: + assert restored.location.address == original.location.address, label + + +def test_round_trip_without_literal_arm_keeps_bare_iri(store): + """With no literal arm a bare IRI is unambiguous, so it stays compact.""" + p = Person(id="ex:p1", employer="ex:acme") + dumped = p.model_dump(exclude_none=True) + assert dumped["employer"] == "ex:acme" # not boxed + restored = Person(**dumped) + assert isinstance(restored.employer, Org) + assert restored.employer.name == "ACME" + + +def test_round_trip_list_of_links(store): + p = Person(id="ex:p1", knows=["ex:p2"], friends=["ex:p2"]) + restored = Person(**p.model_dump(exclude_none=True)) + assert [x.id for x in restored.knows] == ["ex:p2"] + assert [x.id for x in restored.friends] == ["ex:p2"] + assert isinstance(restored.knows[0], Person) + + +def test_equality_is_independent_of_resolution(store): + """Reading a link caches it in __dict__; that must not change equality.""" + a = Person(id="ex:p1", knows=["ex:p2"]) + b = Person(id="ex:p1", knows=["ex:p2"]) + _ = a.knows # resolve on one side only + assert a == b + assert Person(id="ex:p1", knows=["ex:p2"]) != Person(id="ex:p1") + assert Person(id="ex:p1") != Person(id="ex:other") + + +def test_unset_and_explicit_empty_are_distinct(store): + """Unset contributes nothing; an explicit [] is a statement and round-trips.""" + unset = Person(id="ex:u").model_dump(exclude_none=True) + empty = Person(id="ex:e", knows=[]).model_dump(exclude_none=True) + assert "knows" not in unset + assert empty["knows"] == [] + assert Person(**empty).model_dump(exclude_none=True)["knows"] == [] + + +def test_unset_to_many_reads_as_empty_list(store): + """A non-Optional list annotation must not hand back None.""" + p = Person(id="ex:u") + assert p.knows == [] + assert p.employer is None # to-one keeps None, and keeps "| None" + + +def _required(model_cls) -> list: + schema = model_cls.model_json_schema() + if "properties" in schema: + return schema.get("required", []) + return schema["$defs"][model_cls.__name__].get("required", []) + + +def _properties(model_cls) -> dict: + schema = model_cls.model_json_schema() + if "properties" in schema: + return schema["properties"] + return schema["$defs"][model_cls.__name__]["properties"] + + +def test_range_is_derived_from_the_annotation(): + """Presence of ``x-oold-range`` is what makes a property a link, so the + annotation has to put it there - otherwise the recommended declaration + emits a schema that does not round-trip through code generation, and the + only way to get one is to repeat the target in ``OoldField(range=...)``.""" + props = _properties(Person) + assert props["knows"]["x-oold-range"] == Person.get_cls_iri() + assert props["friends"]["x-oold-range"] == Person.get_cls_iri() + assert props["employer"]["x-oold-range"] == Org.get_cls_iri() + assert props["location"]["x-oold-range"] == Location.get_cls_iri() + # the marker was a stand-in for the range; it goes once the range is there + assert "x-oold-link" not in props["knows"] + + +def test_an_explicit_range_is_not_overwritten(): + class Explicit(OoldModel): + id: str + type: str | None = "ex:NExplicit" + target: Link[Org] = OoldField(range="Legacy.json") + + Explicit.model_rebuild() + assert _properties(Explicit)["target"]["x-oold-range"] == "Legacy.json" + + +def test_required_is_a_field_argument_not_the_annotation(): + """Requiredness and the read type are separate questions. A self-link needs + them to differ: father reads as a Person so a walk needs no guard per hop, + while no real dataset can require every person to name one.""" + + class Chain(OoldModel): + id: str + type: str | None = "ex:NChain" + father: Link["Chain"] = OoldField() + manager: Link["Org"] = OoldField(required=True) + + Chain.model_rebuild() + + with pytest.raises(ValueError, match="manager is required"): + Chain(id="ex:c") + c = Chain(id="ex:c", manager="ex:acme") + with pytest.raises(LinkNotResolved): + _ = c.father # optional to supply, still mandatory to read + + props = _properties(Chain) + assert props["manager"]["x-oold-required-iri"] is True + # a plain JSON Schema validator only sees the standard array + assert "manager" in _required(Chain) + assert "father" not in _required(Chain) + + +def test_required_iri_is_still_accepted(): + """Generated packages pass the old spelling.""" + + class Old(OoldModel): + id: str + type: str | None = "ex:NOld" + manager: Link["Org"] = OoldField(required_iri=True) + + Old.model_rebuild() + with pytest.raises(ValueError, match="manager is required"): + Old(id="ex:o") + assert _properties(Old)["manager"]["x-oold-required-iri"] is True + + +def test_a_link_annotation_without_a_default_is_required(): + """No default means required, as it does anywhere else in Python. A link is + never required at the pydantic level - its value is routed out before + validation - so this used to fail with a misleading "Field required" about + a value that had in fact been supplied.""" + + class Bare(OoldModel): + id: str + type: str | None = "ex:NBare" + manager: Link["Org"] + + Bare.model_rebuild() + assert Bare(id="ex:b", manager="ex:acme").link_iris("manager") == "ex:acme" + with pytest.raises(ValueError, match="manager is required"): + Bare(id="ex:b") diff --git a/tests/test_ref.py b/tests/test_ref.py new file mode 100644 index 0000000..8e354d9 --- /dev/null +++ b/tests/test_ref.py @@ -0,0 +1,78 @@ +"""Unit tests for :class:`oold.model._ref.Ref`. + +``Ref`` is the value the descriptor binding stores for a link: it holds either +an unresolved IRI or a resolved object. That indirection is what lets a link be +inspected without resolving it, resolved in batches, and serialised back to an +IRI. +""" + +import pytest + +from oold.backend.document_store import SimpleDictDocumentStore +from oold.backend.interface import SetResolverParam, set_resolver +from oold.model._ref import OoldModel, Ref + +RESOLVE_CALLS = [] + + +class CountingStore(SimpleDictDocumentStore): + def resolve_iris(self, iris): + RESOLVE_CALLS.append(list(iris)) + return super().resolve_iris(iris) + + +class Target(OoldModel): + id: str + label: str | None = None + + +@pytest.fixture() +def store(): + RESOLVE_CALLS.clear() + s = CountingStore() + s.store_json_dicts({"ex:t1": {"id": "ex:t1", "label": "one"}}) + set_resolver(SetResolverParam(iri="ex", resolver=s)) + return s + + +def test_holds_an_unresolved_iri(store): + ref = Ref(iri="ex:t1", target=Target) + assert ref.iri == "ex:t1" + assert ref.resolved is False + assert RESOLVE_CALLS == [] # inspecting must not resolve + + +def test_resolves_on_demand_and_caches(store): + ref = Ref(iri="ex:t1", target=Target) + obj = ref.resolve() + assert isinstance(obj, Target) and obj.label == "one" + assert ref.resolved is True + ref.resolve() + assert len(RESOLVE_CALLS) == 1 # second call served from the cache + + +def test_holds_an_object_and_derives_its_iri(): + ref = Ref(obj=Target(id="ex:t9", label="nine")) + assert ref.resolved is True + assert ref.iri == "ex:t9" # taken from the object + + +def test_attribute_access_delegates(store): + ref = Ref(iri="ex:t1", target=Target) + assert ref.label == "one" # resolves, then reads through + + +def test_equality_and_hash_are_by_iri(): + assert Ref(iri="ex:t1") == Ref(iri="ex:t1") + assert Ref(iri="ex:t1") != Ref(iri="ex:t2") + assert len({Ref(iri="ex:t1"), Ref(iri="ex:t1")}) == 1 + + +def test_missing_target_raises(store): + with pytest.raises(KeyError): + Ref(iri="ex:absent", target=Target).resolve() + + +def test_aresolve_is_available(): + """Async resolution exists as a handle; the in-repo backends are sync.""" + assert callable(Ref(iri="ex:t1").aresolve) diff --git a/tests/test_register_type.py b/tests/test_register_type.py new file mode 100644 index 0000000..bb3cada --- /dev/null +++ b/tests/test_register_type.py @@ -0,0 +1,59 @@ +"""Tests for the public type-registry API. + +Downstream currently imports the private ``_types`` mapping and writes into it +(7 sites across the generated packages and applications), because no public +entry point existed. These functions are that entry point; ``_types`` stays as +the live mapping so existing callers keep working. +""" + +import pytest + +from oold.model import ( + LinkedBaseModel, + _types, + get_registered_type, + register_type, + registered_types, +) + + +class Thing(LinkedBaseModel): + id: str + type: str | None = "ex:RegThing" + + +def test_classes_register_themselves_on_creation(): + assert get_registered_type("ex:RegThing") is Thing + + +def test_register_type_with_explicit_iri(): + """The dynamic-class case that forced downstream to poke at _types.""" + dyn = type("RegDyn", (Thing,), {}) + register_type(dyn, "ex:RegAlias") + assert get_registered_type("ex:RegAlias") is dyn + + +def test_register_type_defaults_to_get_cls_iri(): + register_type(Thing) # idempotent + assert get_registered_type("ex:RegThing") is Thing + + +def test_register_type_accepts_a_list_of_iris(): + dyn = type("RegMulti", (Thing,), {}) + register_type(dyn, ["ex:RegA", "ex:RegB"]) + assert get_registered_type("ex:RegA") is dyn + assert get_registered_type("ex:RegB") is dyn + + +def test_registry_identity_is_shared(): + """Resolution reads this mapping, so identity matters, not a copy.""" + assert registered_types() is _types + + +def test_unknown_iri_returns_none(): + assert get_registered_type("ex:NeverRegistered") is None + + +def test_class_without_iri_raises_clearly(): + with pytest.raises(ValueError, match="no type IRI"): + register_type(type("RegNoIri", (object,), {})) diff --git a/tests/test_sparql_query.py b/tests/test_sparql_query.py new file mode 100644 index 0000000..3d5b674 --- /dev/null +++ b/tests/test_sparql_query.py @@ -0,0 +1,141 @@ +"""Translating the query DSL into SPARQL. + +The DSL builds a ``Condition`` / ``Query`` from ``Model.field == value`` and +friends. Two things consume it: ``apply_operator``, which filters objects already +in memory, and the SPARQL resolvers, which have to ask a triple store the same +question. The check that matters is that they agree - so every case here asserts +the SPARQL answer against the in-memory one over the same data, rather than +against a hand-written expectation. + +Runs against ``LocalSparqlResolver`` (an in-process rdflib graph), so no network. +""" + +import pytest +from pydantic import ConfigDict +from rdflib import Graph + +from oold.backend.interface import ( + ComparisonOperator, + Condition, + Query, + QueryParam, + SetResolverParam, + apply_operator, + set_resolver, +) +from oold.backend.sparql import LocalSparqlBackend, _translate +from oold.model._descriptor import LinkedBaseModel + +EX = "https://sparqltest.example/" +XSD_INT = "http://www.w3.org/2001/XMLSchema#integer" + + +class Person(LinkedBaseModel): + model_config = ConfigDict( + json_schema_extra={ + "@context": { + "id": "@id", + "type": "@type", + # full IRIs, no prefix: a prefix would make compaction rewrite + # the ids too, and the comparison below is about the result set + "name": {"@id": EX + "name"}, + "age": {"@id": EX + "age", "@type": XSD_INT}, + }, + "iri": EX + "Person", + } + ) + id: str + type: str | None = EX + "Person" + name: str | None = None + age: int | None = None + + def get_iri(self): + return self.id + + +# full IRIs, not prefixed: LocalSparqlBackend hardcodes a single "ex:" prologue +PEOPLE = [ + Person(id=EX + "alice", name="Alice", age=30), + Person(id=EX + "bob", name="Bob", age=45), + Person(id=EX + "carol", name="Carol", age=45), +] + + +@pytest.fixture +def backend(): + store = LocalSparqlBackend(graph=Graph()) + store.store_jsonld_dicts({p.get_iri(): p.to_jsonld() for p in PEOPLE}) + set_resolver(SetResolverParam(iri="https", resolver=store)) + return store + + +def _in_memory(condition) -> set[str]: + """What apply_operator says, over the same objects.""" + + def matches(person, node) -> bool: + if isinstance(node, Query): + assert node.operator == "and" + return matches(person, node.op1) and matches(person, node.op2) + return apply_operator(node.operator, getattr(person, node.field, None), node.value) + + return {p.id for p in PEOPLE if matches(p, condition)} + + +def _via_sparql(backend, condition) -> set[str]: + result = backend.query(QueryParam(query=condition, model_cls=Person)) + return {node.id for node in result.nodes.values() if node is not None} + + +@pytest.mark.parametrize( + "operator,field,value", + [ + (ComparisonOperator.EQ, "name", "Bob"), + (ComparisonOperator.NE, "name", "Bob"), + (ComparisonOperator.EQ, "age", 45), + (ComparisonOperator.LT, "age", 45), + (ComparisonOperator.LE, "age", 45), + (ComparisonOperator.GT, "age", 30), + (ComparisonOperator.GE, "age", 30), + ], +) +def test_sparql_agrees_with_the_in_memory_filter(backend, operator, field, value): + condition = Condition(field=field, operator=operator, value=value) + expected = _in_memory(condition) + assert expected, "the fixture should exercise a non-empty result" + assert _via_sparql(backend, condition) == expected + + +def test_conjunction(backend): + condition = Query( + op1=Condition(field="age", operator=ComparisonOperator.EQ, value=45), + operator="and", + op2=Condition(field="name", operator=ComparisonOperator.EQ, value="Bob"), + ) + assert _via_sparql(backend, condition) == _in_memory(condition) == {EX + "bob"} + + +def test_the_model_context_decides_the_predicate_and_the_literal(): + """Not a second reading of the context - it is expanded, like the payload.""" + patterns = _translate(Condition(field="age", operator=ComparisonOperator.GT, value=40), Person, [0]) + assert f"<{EX}age>" in patterns + assert f'"40"^^<{XSD_INT}>' in patterns + + +def test_an_untranslatable_operator_raises_rather_than_guessing(backend): + condition = Query( + op1=Condition(field="name", operator=ComparisonOperator.EQ, value="Bob"), + operator="or", + op2=Condition(field="name", operator=ComparisonOperator.EQ, value="Alice"), + ) + with pytest.raises(NotImplementedError, match="or"): + backend.query(QueryParam(query=condition, model_cls=Person)) + + +def test_unmapped_field_raises(backend): + with pytest.raises(ValueError, match="not mapped"): + backend.query( + QueryParam( + query=Condition(field="nope", operator=ComparisonOperator.EQ, value="x"), + model_cls=Person, + ) + ) diff --git a/tests/test_typing.py b/tests/test_typing.py new file mode 100644 index 0000000..5dda052 --- /dev/null +++ b/tests/test_typing.py @@ -0,0 +1,77 @@ +"""The link and query APIs must keep their static types. + +``tests/typing/`` states them with ``assert_type``; pyright and ty verify them. +Both are run over the whole directory - the contract has to hold in either. + +Both checkers are pointed at the interpreter running the tests rather than at +whatever they would discover themselves. An environment without pydantic does not +fail honestly: ty resolves the imports to ``Unknown``, reports a spurious +``conflicting-metaclass`` and passes every ``assert_type`` vacuously, while +pyright reports ``Expected no type arguments`` on ``Model[...]``. Both look like +results and are not. + +Each checker is skipped when it is not installed, so the suite stays runnable +without a node toolchain or ty on PATH. +""" + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +PROBE_DIR = Path(__file__).parent / "typing" +REPO_ROOT = Path(__file__).parent.parent + + +def _ty_command() -> list[str] | None: + direct = shutil.which("ty") + if direct: + return [direct] + # uv ships ty on demand; without this the test skips silently on machines + # where ty is only ever invoked through uvx + uvx = shutil.which("uvx") + return [uvx, "ty"] if uvx else None + + +def _pyright_command() -> list[str] | None: + direct = shutil.which("pyright") + if direct: + return [direct] + uvx = shutil.which("uvx") + if uvx: + return [uvx, "pyright"] + npx = shutil.which("npx") + return [npx, "--no-install", "pyright"] if npx else None + + +def test_pyright_static_types(): + command = _pyright_command() + if command is None: + pytest.skip("pyright not installed") + proc = subprocess.run( # noqa: S603 + [*command, "--project", str(PROBE_DIR), "--pythonpath", sys.executable, "--outputjson"], + capture_output=True, + text=True, + ) + if not proc.stdout.strip(): + pytest.skip(f"pyright unavailable: {proc.stderr[-300:]}") + diagnostics = json.loads(proc.stdout)["generalDiagnostics"] + problems = [d for d in diagnostics if d["severity"] in ("error", "warning")] + assert not problems, "\n".join(f"{d['file']}:{d['range']['start']['line'] + 1} {d['message']}" for d in problems) + + +def test_ty_static_types(): + """The same contract has to hold in ty - it is what downstream uses.""" + command = _ty_command() + if command is None: + pytest.skip("ty not installed") + proc = subprocess.run( # noqa: S603 + [*command, "check", "--python", sys.prefix, "tests/typing"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + assert proc.returncode == 0, proc.stdout[-3000:] or proc.stderr[-3000:] diff --git a/tests/typing/links.py b/tests/typing/links.py new file mode 100644 index 0000000..087bb78 --- /dev/null +++ b/tests/typing/links.py @@ -0,0 +1,76 @@ +"""Static contract of a link field - checked by pyright and ty. + +Never imported at runtime; ``tests/test_typing.py`` runs both checkers over it +and fails on any diagnostic. + +A link has two types, and one annotation cannot state both: what you read is a +resolved object, what you may write is that object *or* a reference to it - an +IRI string, or a JSON object still to be constructed. ``Link[T]`` and +``LinkList[T]`` carry both by being descriptors, so a checker takes the +``__init__`` parameter and the assignment type from ``__set__`` and the attribute +type from ``__get__`` (PEP 681). + +Optionality is **declared**, not assumed. ``Link[T]`` reads as ``T`` and the +binding keeps that promise - a reference that cannot be resolved raises rather +than returning a ``None`` the type denies. ``Link[T | None]`` reads as +``T | None``, because absence is then part of the model. That is what lets a +chain of mandatory links be written without a guard at every hop. +""" + +from typing_extensions import assert_type + +from oold.model import Link, LinkedBaseModel, LinkList, LinkResultList, OoldField + + +class Org(LinkedBaseModel): + id: str + name: str | None = None + + +class Entity(LinkedBaseModel): + id: str + name: str | None = None + # mandatory: every reference resolves, or the read raises + owner: Link[Org] = OoldField() + parent: Link["Entity"] = OoldField() + links: LinkList["Entity"] = OoldField() + # optional: absence is data + sponsor: Link["Org | None"] = OoldField() + maybe_links: LinkList["Entity | None"] = OoldField() + # plain spelling: runtime-identical, but a checker only sees list[Entity] + plain: list["Entity"] = OoldField() + + +# -- writes: an object, an IRI or a JSON object are all accepted ------------- +written = Entity( + id="ex:e1", + links=["ex:a", Entity(id="ex:b"), {"id": "ex:c"}], + owner="ex:acme", + sponsor=None, +) +written.links = ["ex:d", {"id": "ex:e"}] +written.owner = {"id": "ex:other"} + +# -- reads: exactly what was declared --------------------------------------- +# A separate instance: ty narrows an attribute to the assigned type after a +# write, which would otherwise mask what __get__ declares. +read = Entity(id="ex:e2") +assert_type(read.owner, Org) +assert_type(read.links, LinkResultList[Entity]) +assert_type(read.links[0], Entity) +assert_type(read.links[0].name, str | None) + +# the point of declaring a link mandatory: chaining needs no guard per hop +assert_type(read.parent.parent.parent, Entity) +assert_type(read.parent.parent.owner.name, str | None) + +# declared optional, so the guard is required - and warranted +assert_type(read.sponsor, Org | None) +assert_type(read.maybe_links[0], Entity | None) +sponsor = read.sponsor +if sponsor is not None: + assert_type(sponsor.name, str | None) + +# The plain spelling reads as declared - which is why an IRI cannot be assigned +# to it statically, and why it cannot express either promise. +assert_type(read.plain, list[Entity]) diff --git a/tests/typing/pyrightconfig.json b/tests/typing/pyrightconfig.json new file mode 100644 index 0000000..23cdd32 --- /dev/null +++ b/tests/typing/pyrightconfig.json @@ -0,0 +1,9 @@ +{ + "include": [ + "." + ], + "extraPaths": [ + "../../src" + ], + "typeCheckingMode": "basic" +} diff --git a/tests/typing/query_dsl.py b/tests/typing/query_dsl.py new file mode 100644 index 0000000..f4c8eba --- /dev/null +++ b/tests/typing/query_dsl.py @@ -0,0 +1,38 @@ +"""Static contract of the class-level query API - checked by pyright and ty. + +Never imported at runtime; ``tests/test_typing.py`` runs both checkers over it +and fails on any diagnostic. ``Model[...]`` is typed by overloads on the +metaclass ``__getitem__``, which both checkers resolve. + +The condition expression itself stays untyped: ``Entity.name`` is a declared +field, so pydantic's ``dataclass_transform`` makes the annotation authoritative +for class-level access and the ``FieldProxy`` returned at runtime is invisible, +leaving ``Entity.name == "x"`` as ``bool``. The subscript overloads accept +``bool`` for exactly that reason - the result type is right even though the +argument type is not. +""" + +from typing_extensions import assert_type + +from oold.model import LinkedBaseModel, LinkResultList + + +class Entity(LinkedBaseModel): + id: str + name: str | None = None + + +# a single IRI yields one instance, a condition or a list of IRIs yields a list +assert_type(Entity["ex:e1"], Entity | None) +assert_type(Entity[Entity.name == "x"], LinkResultList[Entity] | None) +assert_type(Entity[["ex:e1", "ex:e2"]], LinkResultList[Entity] | None) + +many = Entity[Entity.name == "x"] +if many is not None: + # indexing and filtering keep the item type. Elements are not optional: + # a query answers with what it found, so an IRI it could not place is + # dropped rather than kept as a None + assert_type(many[0], Entity) + assert_type(many[0:2], LinkResultList[Entity]) + assert_type(many[Entity.name == "y"], LinkResultList[Entity]) + assert_type(many[0].name, str | None)