From 026e30cffc198dbebb5a6719646854c3fd5dc4b5 Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Thu, 17 Sep 2026 00:42:01 -0700 Subject: [PATCH] Add compatible claim evidence assessments and study-source types --- docs/EVIDENCE_ASSESSMENT.md | 65 ++++++++++ .../schema/cellstructuremech.yaml | 44 +++++++ tests/test_evidence_assessment.py | 120 ++++++++++++++++++ 3 files changed, 229 insertions(+) create mode 100644 docs/EVIDENCE_ASSESSMENT.md create mode 100644 tests/test_evidence_assessment.py diff --git a/docs/EVIDENCE_ASSESSMENT.md b/docs/EVIDENCE_ASSESSMENT.md new file mode 100644 index 00000000..fca0019d --- /dev/null +++ b/docs/EVIDENCE_ASSESSMENT.md @@ -0,0 +1,65 @@ +# Claim evidence assessment + +`supports` states how the cited evidence bears on the **specific attached claim**. +`evidence_source` identifies the study or data-source type that produced it. +Neither field is a confidence score, a citation identifier, an ECO evidence code, +or a record of how a curator or language model found the source. + +Applies to: `EvidenceItem`. + +## Support values + +- **SUPPORT**: The cited evidence supports the attached claim. +- **REFUTE**: The cited evidence contradicts the attached claim. +- **PARTIAL**: The cited evidence supports only part of the claim or only under stated conditions. +- **NO_EVIDENCE**: The citation provides no relevant evidence for the attached claim; not proof of a negative result. +- **WRONG_STATEMENT**: The attached claim was assessed as incorrect; retain the assessment and its explanation. + +Omitted optional assessments mean **unassessed**, not SUPPORT or OTHER. Existing +required assessments remain required. Explain PARTIAL, REFUTE, NO_EVIDENCE and +WRONG_STATEMENT in the evidence's existing `notes` or `explanation` field. +A failed experiment is not automatically NO_EVIDENCE: it may refute a claim or +support a claim about the absence of an effect in a particular setting. + +## Study and data-source values + +- **FIELD_STUDY**: Observation or measurement made in a natural system. +- **MESOCOSM**: Experiment in a controlled enclosure representing a natural system. +- **LABORATORY**: Controlled laboratory study; use IN_VITRO or IN_VIVO when that distinction is known. +- **IN_VITRO**: Experiment outside an intact organism, such as a culture or biochemical assay. +- **IN_VIVO**: Experiment or observation in an intact living organism. +- **COMPUTATIONAL**: Model, simulation, prediction, or computational analysis providing the cited evidence. +- **META_ANALYSIS**: Systematic synthesis or quantitative analysis of results across studies. +- **REVIEW**: Narrative review or expert synthesis of published work. +- **REMOTE_SENSING**: Satellite, aerial, or other remotely sensed observations. +- **LONG_TERM_MONITORING**: Repeated observations from a long-term monitoring programme. +- **EXPERT_OPINION**: Expert judgement or consensus rather than a primary experimental result. +- **DATABASE**: Assertion or observation obtained from a database or curated data resource. +- **OTHER**: A known study or data-source type outside these categories; explain it in notes or explanation. + +Choose the most specific defensible category. IN_VITRO and IN_VIVO preserve the +existing microbial-study distinction. LABORATORY is available when the study +is known to be laboratory-based but that distinction is not established. +REVIEW and META_ANALYSIS are separate; DATABASE identifies a database assertion +without promoting it to primary literature. Unknown source types stay omitted +where optional; OTHER requires a known type that does not fit the vocabulary. + +## Compatibility and curation + +No existing citation format, required field, evidence type, or provenance slot +has been removed. Newly introduced assessment fields are optional, with no +default and no automatic backfill. Populate them only after checking the source +against its attached claim. A citation's mere presence must not be interpreted +as positive support when its assessment is REFUTE or NO_EVIDENCE. + +These fields belong to primary claim evidence. The imported shared +`SupportingReference.evidence_source` remains legacy free text describing where +a quote was obtained (abstract, full_text, figure, etc.). It has not been narrowed +to this enum. Likewise, typed strain/genome-link provenance and existing +`source`, `evidence_type` and `retrieved_on` fields retain their own meanings. + +## Provenance + +Motivated by [EcoMech's evidence schema](https://github.com/diatomsRcool/ecomech/blob/5d3f55467f3caaa6e88eb7f33037e4a8103873a7/src/ecomech/schema/ecomech.yaml) +and the existing CommunityMech, CultureMech and MediaIngredientMech assessment +models. Fleet agreement: [CLAW evidence contract](https://github.com/CultureBotAI/culturebotai-claw/blob/26d083901922650688aaa4937c34c7e68856183a/docs/standards/evidence_assessment.yaml). diff --git a/src/cellstructuremech/schema/cellstructuremech.yaml b/src/cellstructuremech/schema/cellstructuremech.yaml index 47e32158..a215bed0 100644 --- a/src/cellstructuremech/schema/cellstructuremech.yaml +++ b/src/cellstructuremech/schema/cellstructuremech.yaml @@ -749,6 +749,17 @@ classes: EvidenceItem: description: A literature or database citation supporting a claim. attributes: + supports: + description: >- + How this evidence bears on the attached claim, not confidence or + evidence quality. Omission means unassessed, never implicit SUPPORT. + range: SupportLevelEnum + evidence_source: + description: >- + Study or data-source type behind the cited evidence, not the curation + method or the location of a quote. Omit when unknown; never infer + from a PMID, DOI, or whether an LLM assisted curation. + range: EvidenceSourceEnum reference: description: PMID:..., DOI:..., or an opaque URL. required: true @@ -877,6 +888,39 @@ classes: enums: + EvidenceSourceEnum: + description: >- + Study or data-source type providing a claim's evidence. This is independent + of supports, citation identifiers, quotation location, and curation method. + permissible_values: + FIELD_STUDY: + description: Observation or measurement made in a natural system. + MESOCOSM: + description: Experiment in a controlled enclosure representing a natural system. + LABORATORY: + description: Controlled laboratory study; use IN_VITRO or IN_VIVO when that distinction is known. + IN_VITRO: + description: Experiment outside an intact organism, such as a culture or biochemical assay. + IN_VIVO: + description: Experiment or observation in an intact living organism. + COMPUTATIONAL: + description: Model, simulation, prediction, or computational analysis providing the cited evidence. + META_ANALYSIS: + description: Systematic synthesis or quantitative analysis of results across studies. + REVIEW: + description: Narrative review or expert synthesis of published work. + REMOTE_SENSING: + description: Satellite, aerial, or other remotely sensed observations. + LONG_TERM_MONITORING: + description: Repeated observations from a long-term monitoring programme. + EXPERT_OPINION: + description: Expert judgement or consensus rather than a primary experimental result. + DATABASE: + description: Assertion or observation obtained from a database or curated data resource. + OTHER: + description: A known study or data-source type outside these categories; explain it in notes or explanation. + + StructuralDatasetTypeEnum: description: >- Shared dataset types plus CellStructureMech structural data. Shared diff --git a/tests/test_evidence_assessment.py b/tests/test_evidence_assessment.py new file mode 100644 index 00000000..f6582fe4 --- /dev/null +++ b/tests/test_evidence_assessment.py @@ -0,0 +1,120 @@ +"""Claim assessments are typed, backward-compatible and never defaulted.""" + +from pathlib import Path + +import pytest +from linkml.validator import Validator +from linkml.validator.plugins import JsonschemaValidationPlugin +from linkml_runtime.utils.schemaview import SchemaView + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = ROOT / "src/cellstructuremech/schema/cellstructuremech.yaml" +CLASSES = ["EvidenceItem"] +SUPPORT = {"SUPPORT", "REFUTE", "PARTIAL", "NO_EVIDENCE", "WRONG_STATEMENT"} +SOURCES = { + "FIELD_STUDY", + "MESOCOSM", + "LABORATORY", + "IN_VITRO", + "IN_VIVO", + "COMPUTATIONAL", + "META_ANALYSIS", + "REVIEW", + "REMOTE_SENSING", + "LONG_TERM_MONITORING", + "EXPERT_OPINION", + "DATABASE", + "OTHER", +} +REQUIRED = {"EvidenceItem": {"supports": False, "evidence_source": False}} + + +@pytest.fixture(scope="module") +def assessment_view(): + return SchemaView(str(SCHEMA)) + + +@pytest.fixture(scope="module") +def assessment_validator(): + return Validator(str(SCHEMA), validation_plugins=[JsonschemaValidationPlugin(closed=True)]) + + +def baseline(view, cls): + values = { + "reference": "PMID:12345678", + "snippet": "A verbatim test fixture passage.", + "explanation": "Fixture assessment, not a curated scientific assertion.", + "source": "fixture", + "supports": "SUPPORT", + "evidence_source": "IN_VITRO", + } + result = {} + for slot in view.class_induced_slots(cls): + if slot.required: + if slot.name in values: + result[slot.name] = values[slot.name] + elif slot.range in view.all_enums(): + result[slot.name] = next(iter(view.get_enum(slot.range).permissible_values)) + else: + raise AssertionError(f"Add an explicit fixture for {cls}.{slot.name}") + return result + + +@pytest.mark.parametrize("cls", CLASSES) +def test_assessments_preserve_legacy_requiredness_and_have_no_default( + assessment_view, assessment_validator, cls +): + for name in ("supports", "evidence_source"): + slot = assessment_view.induced_slot(name, cls) + assert bool(slot.required) == REQUIRED[cls][name] + assert slot.ifabsent is None + expected = SUPPORT if name == "supports" else SOURCES + assert set(assessment_view.get_enum(slot.range).permissible_values) == expected + old = baseline(assessment_view, cls) + assert not list(assessment_validator.iter_results(old, target_class=cls)) + + +@pytest.mark.parametrize("cls", CLASSES) +@pytest.mark.parametrize("support", sorted(SUPPORT)) +def test_all_support_assessments_validate(assessment_view, assessment_validator, cls, support): + data = {**baseline(assessment_view, cls), "supports": support, "evidence_source": "DATABASE"} + assert not list(assessment_validator.iter_results(data, target_class=cls)) + + +@pytest.mark.parametrize("cls", CLASSES) +@pytest.mark.parametrize("source", sorted(SOURCES)) +def test_study_types_validate_without_changing_support(assessment_view, assessment_validator, cls, source): + data = {**baseline(assessment_view, cls), "supports": "REFUTE", "evidence_source": source} + assert not list(assessment_validator.iter_results(data, target_class=cls)) + assert data["supports"] == "REFUTE" + + +@pytest.mark.parametrize("cls", CLASSES) +@pytest.mark.parametrize( + "field,value", + [ + ("supports", "HIGH_CONFIDENCE"), + ("supports", "support"), + ("evidence_source", "abstract"), + ("evidence_source", "LLM_ASSISTED"), + ("evidence_source", "PMID:123"), + ], +) +def test_rejects_conflating_assessment_source_and_curation( + assessment_view, assessment_validator, cls, field, value +): + valid = {**baseline(assessment_view, cls), "supports": "PARTIAL", "evidence_source": "LABORATORY"} + assert not list(assessment_validator.iter_results(valid, target_class=cls)) # positive control + invalid = {**valid, field: value} + assert list(assessment_validator.iter_results(invalid, target_class=cls)) + + +@pytest.mark.parametrize("cls", CLASSES) +def test_unknown_fields_still_fail_closed_validation(assessment_view, assessment_validator, cls): + data = {**baseline(assessment_view, cls), "invented_evidence_field": "not allowed"} + assert list(assessment_validator.iter_results(data, target_class=cls)) + + +def test_shared_reference_location_keeps_its_legacy_meaning(assessment_validator): + data = {"reference": "PMID:12345678", "supports": "PARTIAL", "evidence_source": "abstract"} + assert not list(assessment_validator.iter_results(data, target_class="SupportingReference"))