From 137c3d6f9fbf0f32e8b15d67d41036571f943735 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Sun, 1 Nov 2026 15:58:20 +0200 Subject: [PATCH 1/5] feat(sfp): implement operation admission --- .../assert-operation-admission/action.yml | 51 ++ .../assert-operation-admission/assert.py | 279 +++++++ .../create-fixture/fixtures/manifest.py | 65 ++ .github/actions/provider/start/action.yml | 7 + .../operations/admission/action.yml | 62 ++ .github/workflows/rpm-installed-test.yml | 8 + docs/ci-cd.md | 12 +- docs/configuration.md | 8 + docs/index.md | 10 +- docs/sfp-operation-admission.md | 107 +++ mkdocs.yml | 1 + packaging/rpm/SOURCES/stateforge-agent.toml | 4 + pyproject.toml | 2 + src/stateforge_agent/_config/loading.py | 44 +- src/stateforge_agent/_config/models.py | 4 + src/stateforge_agent/_journal/base.py | 11 + src/stateforge_agent/_journal/sqlite.py | 17 + .../_journal/sqlite_operation_admissions.py | 473 ++++++++++++ src/stateforge_agent/_runtime/daemon.py | 19 + src/stateforge_agent/_sfp/__init__.py | 38 + .../_sfp/_runtime/sessions.py | 124 ++++ .../_sfp/_services/manifest/declaration.py | 38 + .../_services/operation_admission/__init__.py | 35 + .../_services/operation_admission/client.py | 279 +++++++ .../operation_admission/coordinator.py | 582 +++++++++++++++ .../_services/operation_admission/models.py | 66 ++ .../_services/operation_admission/store.py | 120 +++ tests/test_configuration.py | 13 + tests/test_daemon_provider_events.py | 4 + tests/test_journal_registry.py | 12 + tests/test_operation_admission.py | 687 ++++++++++++++++++ tests/test_operation_admission_store.py | 214 ++++++ tests/test_rpm_automation.py | 58 +- uv.lock | 42 ++ 34 files changed, 3490 insertions(+), 6 deletions(-) create mode 100644 .github/actions/agent/assert-operation-admission/action.yml create mode 100644 .github/actions/agent/assert-operation-admission/assert.py create mode 100644 .github/actions/rpm/test-installed/operations/admission/action.yml create mode 100644 docs/sfp-operation-admission.md create mode 100644 src/stateforge_agent/_journal/sqlite_operation_admissions.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_admission/__init__.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_admission/client.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_admission/models.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_admission/store.py create mode 100644 tests/test_operation_admission.py create mode 100644 tests/test_operation_admission_store.py diff --git a/.github/actions/agent/assert-operation-admission/action.yml b/.github/actions/agent/assert-operation-admission/action.yml new file mode 100644 index 0000000..2be9428 --- /dev/null +++ b/.github/actions/agent/assert-operation-admission/action.yml @@ -0,0 +1,51 @@ +name: Assert Agent Operation Admission +description: Exercise the RPM-installed Agent client against a real Provider. + +inputs: + container-name: + description: Fedora container running the installed Agent and Provider. + required: true + provider-instance-id: + description: Provider installation identity published by the fixture. + required: true + provider-state-name: + description: Fixture state directory containing operation evidence. + required: false + default: provider-a + evidence-directory: + description: Host directory receiving sanitized admission evidence. + required: true + +runs: + using: composite + steps: + - name: Exercise installed Agent Operation Admission + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + CONTAINER_NAME: ${{ inputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + PROVIDER_INSTANCE_ID: ${{ inputs.provider-instance-id }} + PROVIDER_STATE_NAME: ${{ inputs.provider-state-name }} + run: | + set -euo pipefail + mkdir -p "${EVIDENCE_DIRECTORY}" + verifier_directory="/opt/stateforge-agent-tests/operation-admission" + verifier="${verifier_directory}/assert.py" + state_directory="/var/lib/stateforge-provider-fixtures/${PROVIDER_STATE_NAME}" + docker exec "${CONTAINER_NAME}" mkdir -p "${verifier_directory}" + docker cp "${ACTION_PATH}/assert.py" \ + "${CONTAINER_NAME}:${verifier}" + set +e + docker exec "${CONTAINER_NAME}" /usr/bin/python3 "${verifier}" \ + --provider-instance-id "${PROVIDER_INSTANCE_ID}" \ + --state-directory "${state_directory}" \ + > "${EVIDENCE_DIRECTORY}/operation-admission.json" + status=$? + set -e + docker exec "${CONTAINER_NAME}" cat "${state_directory}/events.log" \ + > "${EVIDENCE_DIRECTORY}/operation-events.txt" 2>&1 || true + docker exec "${CONTAINER_NAME}" sha256sum \ + "${state_directory}/target-state.txt" \ + > "${EVIDENCE_DIRECTORY}/target-sha256.txt" 2>&1 || true + exit "${status}" diff --git a/.github/actions/agent/assert-operation-admission/assert.py b/.github/actions/agent/assert-operation-admission/assert.py new file mode 100644 index 0000000..3999ab4 --- /dev/null +++ b/.github/actions/agent/assert-operation-admission/assert.py @@ -0,0 +1,279 @@ +"""Exercise Operation Admission from the RPM-installed Agent package.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import sys +import time +from datetime import UTC, datetime, timedelta +from importlib.metadata import version +from pathlib import Path +from uuid import UUID, uuid4 + +from stateforge_agent._journal import SqliteJournal +from stateforge_agent._sfp import ( + OperationAdmissionDisposition, + OperationAdmissionPreconditionError, + OperationAdmissionRequest, + ProviderAvailabilityState, + ProviderSessionManager, + SfpCapabilityAssessmentRequestError, + SfpVersion, +) +from stateforge_agent._sfp._runtime.descriptor import load_provider_descriptor +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCoordinator, +) + +_AGENT_JOURNAL = Path("/var/lib/stateforge-agent/journal/agent.db") +_AGENT_UNIT = "stateforge-agent.service" +_DISCOVERY_DIRECTORY = Path("/run/stateforge-agent/providers") +_OPERATION = "stateforge.test/inspect@1" +_OPERATION_ATTEMPT_PREFIX = "rpc:operation-admission:" +_EXPECTED_ATTEMPTS = 2 + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--provider-instance-id", type=UUID, required=True) + parser.add_argument("--state-directory", type=Path, required=True) + return parser.parse_args() + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _events(path: Path) -> tuple[str, ...]: + try: + return tuple(path.read_text(encoding="utf-8").splitlines()) + except FileNotFoundError: + return () + + +def _wait_for_operation_event(path: Path) -> None: + """Wait outside the event loop for one deterministic fixture dispatch.""" + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if _events(path).count("operation:inspect") == 1: + return + time.sleep(0.05) + raise AssertionError("Provider did not dispatch exactly one operation") + + +async def _restart_agent() -> None: + """Restart the exact packaged Agent unit under test.""" + process = await asyncio.create_subprocess_exec( + "/usr/bin/systemctl", + "restart", + _AGENT_UNIT, + ) + if await process.wait() != 0: + raise AssertionError("Installed Agent service restart failed") + + +async def _wait_until_available( + manager: ProviderSessionManager, + provider_instance_id: UUID, +) -> None: + async with asyncio.timeout(15): + while True: + status = manager.availability_status(provider_instance_id) + if ( + status is not None + and status.state is ProviderAvailabilityState.AVAILABLE + ): + return + await asyncio.sleep(0.05) + + +async def _manager( + journal: SqliteJournal, +) -> tuple[ProviderSessionManager, OperationAdmissionCoordinator]: + store = journal.operation_admissions(maximum_unresolved_attempts=32) + await store.initialize() + coordinator = OperationAdmissionCoordinator( + store, + attempts=_EXPECTED_ATTEMPTS, + retry_initial_delay_seconds=0.05, + ) + manager = ProviderSessionManager( + agent_runtime_id=uuid4(), + agent_software_version=version("stateforge-agent"), + supported_sfp_versions=(SfpVersion(major=1, minor=0),), + capability_assessment_retry_initial_delay_seconds=0.05, + operation_admission=coordinator, + ) + return manager, coordinator + + +async def _require_local_rejections( + manager: ProviderSessionManager, + provider_instance_id: UUID, + event_log: Path, +) -> None: + attempts_before = tuple( + event + for event in _events(event_log) + if event.startswith(_OPERATION_ATTEMPT_PREFIX) + ) + requests = ( + OperationAdmissionRequest( + provider_instance_id=provider_instance_id, + operation_identifier="stateforge.test/not-declared@1", + payload={"value": "rejected"}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=5), + execution_context={"scenario": "installed-agent"}, + ), + OperationAdmissionRequest( + provider_instance_id=provider_instance_id, + operation_identifier=_OPERATION, + payload={"unexpected": "field"}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=5), + execution_context={"scenario": "installed-agent"}, + ), + OperationAdmissionRequest( + provider_instance_id=provider_instance_id, + operation_identifier=_OPERATION, + payload={"value": "rejected"}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=5), + execution_context={"scenario": "installed-agent"}, + required_implementation_roles=( + "stateforge.test.profile.v1.not_declared", + ), + ), + ) + for request in requests: + try: + await manager.admit_operation(request) + except ( + OperationAdmissionPreconditionError, + SfpCapabilityAssessmentRequestError, + ): + continue + raise AssertionError("Agent accepted an invalid local operation") + attempts_after = tuple( + event + for event in _events(event_log) + if event.startswith(_OPERATION_ATTEMPT_PREFIX) + ) + if attempts_after != attempts_before: + raise AssertionError("Locally rejected operation reached Provider") + + +async def _run(arguments: argparse.Namespace) -> dict[str, object]: + provider_instance_id: UUID = arguments.provider_instance_id + descriptor = load_provider_descriptor( + _DISCOVERY_DIRECTORY / f"{provider_instance_id}.sfp.json", + discovery_directory=_DISCOVERY_DIRECTORY, + ) + event_log = arguments.state_directory / "events.log" + target = arguments.state_directory / "target-state.txt" + target_before = _digest(target) + + journal = SqliteJournal(_AGENT_JOURNAL) + await journal.open() + manager, coordinator = await _manager(journal) + try: + await manager.establish(descriptor) + await _wait_until_available(manager, provider_instance_id) + await _require_local_rejections( + manager, + provider_instance_id, + event_log, + ) + result = await manager.admit_operation( + OperationAdmissionRequest( + provider_instance_id=provider_instance_id, + operation_identifier=_OPERATION, + payload={"value": "admitted-once"}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=5), + execution_context={"scenario": "installed-agent"}, + ) + ) + attempt = await coordinator.attempt( + provider_instance_id, + result.operation_id, + ) + finally: + await manager.close() + await journal.close() + + if result.disposition is not ( + OperationAdmissionDisposition.RESOLVED_EXISTING + ): + raise AssertionError("Lost response was not resolved idempotently") + if attempt is None or attempt.result != result: + raise AssertionError("Accepted operation was not durably resolved") + if attempt.attempt_count != _EXPECTED_ATTEMPTS: + raise AssertionError("Admission did not use the expected retry count") + + await asyncio.to_thread(_wait_for_operation_event, event_log) + events = _events(event_log) + request_digests = tuple( + event.removeprefix(_OPERATION_ATTEMPT_PREFIX) + for event in events + if event.startswith(_OPERATION_ATTEMPT_PREFIX) + ) + if ( + len(request_digests) != _EXPECTED_ATTEMPTS + or len(set(request_digests)) != 1 + ): + raise AssertionError("Retry changed the immutable wire request") + expected_responses = { + "operation-admission-response:dropped:accepted_new", + "operation-admission-response:returned:resolved_existing", + } + if not expected_responses.issubset(events): + raise AssertionError("Provider did not expose both admission outcomes") + if _digest(target) != target_before: + raise AssertionError("Generic fixture operation mutated its target") + + await _restart_agent() + reopened = SqliteJournal(_AGENT_JOURNAL) + await reopened.open() + reopened_store = reopened.operation_admissions( + maximum_unresolved_attempts=32 + ) + await reopened_store.initialize() + try: + persisted = await reopened_store.get( + provider_instance_id, + result.operation_id, + ) + finally: + await reopened.close() + if persisted is None or persisted.result != result: + raise AssertionError("Agent restart lost the admission record") + + return { + "agent_restart_preserved_record": True, + "agent_version": version("stateforge-agent"), + "attempt_count": attempt.attempt_count, + "disposition": result.disposition.value, + "endpoint_generation": result.endpoint_generation, + "fingerprint": result.request_fingerprint, + "manifest_revision": result.manifest_revision, + "operation_id": str(result.operation_id), + "operation_identifier": _OPERATION, + "provider_instance_id": str(result.provider_instance_id), + "provider_version": version("stateforge-provider"), + "protocol_version": version("stateforge-protocol"), + "request_identity_preserved": True, + "target_sha256": target_before, + } + + +def main() -> None: + """Run the installed-package assertion and emit sanitized evidence.""" + sys.stdout.write( + json.dumps(asyncio.run(_run(_arguments())), indent=2, sort_keys=True) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/.github/actions/provider/create-fixture/fixtures/manifest.py b/.github/actions/provider/create-fixture/fixtures/manifest.py index 339d362..7c6199f 100644 --- a/.github/actions/provider/create-fixture/fixtures/manifest.py +++ b/.github/actions/provider/create-fixture/fixtures/manifest.py @@ -15,6 +15,9 @@ from stateforge_protocol.sfp.capability_assessment.v1 import ( capability_assessment_pb2, ) +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, +) from stateforge_protocol.sfp.trigger_publication.v1 import ( trigger_publication_pb2, ) @@ -32,6 +35,9 @@ from stateforge_provider._services.capability_assessment.service import ( CapabilityAssessmentService, ) +from stateforge_provider._services.operation_admission.service import ( + OperationAdmissionService, +) from stateforge_provider._services.trigger_publication import ( runtime as trigger_publication_runtime, ) @@ -41,6 +47,7 @@ ) _CAPABILITY_ATTEMPT_PREFIX = "rpc:capability-assessment:" +_OPERATION_ADMISSION_ATTEMPT_PREFIX = "rpc:operation-admission:" def _record_event(path: Path, event: str) -> None: @@ -305,6 +312,63 @@ async def assess_capabilities( CapabilityAssessmentService.AssessCapabilities = assess_capabilities +def _install_operation_response_loss_once() -> None: + """Drop the first response only after the real Provider admits work.""" + if ( + os.environ.get( + "STATEFORGE_TEST_OPERATION_DROP_RESPONSE_ONCE", + "false", + ) + != "true" + ): + return + original = OperationAdmissionService.StartOperation + drop_response = True + event_log = Path(os.environ["STATEFORGE_TEST_EVENT_LOG"]) + + async def start_operation( + service: OperationAdmissionService, + request: operation_admission_pb2.StartOperationRequest, + context: grpc.aio.ServicerContext[ + operation_admission_pb2.StartOperationRequest, + operation_admission_pb2.StartOperationResponse, + ], + ) -> operation_admission_pb2.StartOperationResponse: + nonlocal drop_response + digest = hashlib.sha256( + request.SerializeToString(deterministic=True) + ).hexdigest() + response = await original(service, request, context) + disposition = ( + operation_admission_pb2.AdmissionDisposition.Name( + response.disposition + ) + .removeprefix("ADMISSION_DISPOSITION_") + .lower() + ) + _record_event( + event_log, + f"{_OPERATION_ADMISSION_ATTEMPT_PREFIX}{digest}", + ) + if drop_response: + drop_response = False + _record_event( + event_log, + f"operation-admission-response:dropped:{disposition}", + ) + await context.abort( + grpc.StatusCode.UNAVAILABLE, + "Injected response loss after Operation Admission", + ) + _record_event( + event_log, + f"operation-admission-response:returned:{disposition}", + ) + return response + + OperationAdmissionService.StartOperation = start_operation + + def _install_trigger_acknowledgement_observer() -> None: """Record terminal acknowledgements and optionally discard them.""" drop_acknowledgements = ( @@ -377,6 +441,7 @@ def next_publication_id() -> UUID: def main() -> None: """Run the Provider fixture until its service manager stops it.""" _install_capability_unavailable_once() + _install_operation_response_loss_once() _install_trigger_acknowledgement_observer() _install_first_trigger_publication_id() ManifestFixtureProvider( diff --git a/.github/actions/provider/start/action.yml b/.github/actions/provider/start/action.yml index 8935d77..6e641c1 100644 --- a/.github/actions/provider/start/action.yml +++ b/.github/actions/provider/start/action.yml @@ -32,6 +32,10 @@ inputs: description: Make the first real Capability Assessment RPC unavailable. required: false default: "false" + operation-drop-response-once: + description: Drop the first Operation Admission response after dispatch. + required: false + default: "false" drop-trigger-acks-until-restart: description: Discard trigger acknowledgements until the restart marker exists. required: false @@ -71,6 +75,7 @@ runs: PROVIDER_IDENTITY: ${{ inputs.provider-identity }} PROVIDER_NAME: ${{ inputs.provider-name }} PROVIDER_STATE_NAME: ${{ inputs.provider-state-name }} + OPERATION_DROP_RESPONSE_ONCE: ${{ inputs.operation-drop-response-once }} TIMEOUT_SECONDS: ${{ inputs.timeout-seconds }} WHEEL_DIRECTORY: ${{ inputs.wheel-directory }} run: | @@ -163,6 +168,8 @@ runs: "STATEFORGE_TEST_TARGET_PATH=${target_path}" \ --setenv \ "STATEFORGE_TEST_CAPABILITY_UNAVAILABLE_ONCE=${CAPABILITY_UNAVAILABLE_ONCE}" \ + --setenv \ + "STATEFORGE_TEST_OPERATION_DROP_RESPONSE_ONCE=${OPERATION_DROP_RESPONSE_ONCE}" \ --setenv \ "STATEFORGE_TEST_DROP_TRIGGER_ACKS=${DROP_TRIGGER_ACKS}" \ --setenv \ diff --git a/.github/actions/rpm/test-installed/operations/admission/action.yml b/.github/actions/rpm/test-installed/operations/admission/action.yml new file mode 100644 index 0000000..4109475 --- /dev/null +++ b/.github/actions/rpm/test-installed/operations/admission/action.yml @@ -0,0 +1,62 @@ +name: Test installed Operation Admission +description: Verify the RPM Agent against the real Provider Python runtime. + +inputs: + rpm-directory: + description: Directory containing the exact RPM artifact under test. + required: false + default: dist/rpm + evidence-directory: + description: Directory receiving this scenario's evidence. + required: false + default: dist/agent-system-test-evidence/operation-admission + fixture-directory: + description: Directory containing the exact Provider fixture bundle. + required: false + default: dist/provider-fixture + +runs: + using: composite + steps: + - name: Prepare installed Agent + id: setup + uses: ./.github/actions/rpm/test-installed/setup + with: + rpm-directory: ${{ inputs.rpm-directory }} + evidence-directory: ${{ inputs.evidence-directory }} + fixture-directory: ${{ inputs.fixture-directory }} + + - name: Start operation-capable Provider + id: provider + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: provider-a + provider-identity: stateforge.test.operation_admission + operation-drop-response-once: "true" + + - name: Require Agent readiness + uses: ./.github/actions/agent/assert-availability + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + endpoint-generation: ${{ steps.provider.outputs.endpoint-generation }} + state: available + journal-cursor: ${{ steps.provider.outputs.agent-journal-cursor }} + evidence-directory: ${{ inputs.evidence-directory }}/availability + + - name: Exercise Operation Admission through the installed Agent + uses: ./.github/actions/agent/assert-operation-admission + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Preserve evidence and clean up + if: ${{ always() }} + uses: ./.github/actions/rpm/test-installed/cleanup + with: + container-name: ${{ steps.setup.outputs.container-name }} + evidence-directory: ${{ inputs.evidence-directory }} diff --git a/.github/workflows/rpm-installed-test.yml b/.github/workflows/rpm-installed-test.yml index 8764b1b..035a922 100644 --- a/.github/workflows/rpm-installed-test.yml +++ b/.github/workflows/rpm-installed-test.yml @@ -107,6 +107,7 @@ jobs: - capability-malformed - capability-mismatch - capability-limits + - operation-admission - provider-availability - trigger-publication - trigger-storage-failure @@ -153,6 +154,13 @@ jobs: evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} fixture-directory: dist/provider-fixture + - name: Run Operation Admission scenario + if: ${{ matrix.scenario == 'operation-admission' }} + uses: ./.github/actions/rpm/test-installed/operations/admission + with: + evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} + fixture-directory: dist/provider-fixture + - name: Run multiple-Providers scenario if: ${{ matrix.scenario == 'multiple-providers' }} uses: ./.github/actions/rpm/test-installed/lifecycle/multiple-providers diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 3821a37..cab588c 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -26,6 +26,8 @@ operation. │ │ │ └── provider-availability/ │ │ ├── triggers/ │ │ │ └── publication/ +│ │ ├── operations/ +│ │ │ └── admission/ │ │ ├── adversarial/ │ │ │ └── invalid-descriptor/ │ │ └── destructive/ @@ -34,6 +36,7 @@ operation. │ │ ├── assert-availability/ │ │ ├── assert-capability-assessment/ │ │ ├── assert-trigger-publication/ +│ │ ├── assert-operation-admission/ │ │ ├── install/ │ │ ├── start/ │ │ └── collect-evidence/ @@ -119,7 +122,7 @@ Contents permission. The token is scoped to those two repositories, is not persisted by checkout, and expires automatically. A long-lived personal access token is neither required nor accepted by the workflow interface. -The installed-system workflow currently runs eleven isolated scenarios: +The installed-system workflow currently runs twelve isolated scenarios: - smoke: one public-API Provider completes manifest exchange; - capability: the RPM-installed Agent retries one deterministic transient @@ -137,6 +140,10 @@ The installed-system workflow currently runs eleven isolated scenarios: - triggers: a real Provider wheel publishes through both the decorated and callback author APIs; the installed Agent validates and journals the events, survives an Agent service restart, and returns a terminal acknowledgement; +- operations: the installed Agent durably creates one immutable admission + intent against a real Provider Python service, survives a deliberately lost + successful response, retries the same identity, proves exactly-once Provider + dispatch, resolves the existing record, and retains it across Agent restart; - adversarial: malformed discovery data is rejected without stopping the Agent; - destructive: the exact RPM is removed, installed again in a clean environment, and completes manifest exchange after reinstallation. @@ -162,6 +169,7 @@ digest of each generated wheel. | `agent/assert-capability-assessment` | Use the RPM-installed Agent package to collect and verify normalized evidence from a separate real Provider process. | | `agent/assert-capability-boundary` | Exercise timeout, malformed response, stale response-fence, and byte-limit behavior through the RPM-installed client. | | `agent/assert-trigger-publication` | Validate canonical, durable Provider publication evidence in the installed Agent journal. | +| `agent/assert-operation-admission` | Verify local preflight, durable intent, retry identity, real Provider idempotency, exactly-once dispatch, and restart persistence. | | `agent/collect-evidence` | Capture bounded evidence, uninstall the RPM, and clean up. | | `provider/create-fixture` | Invoke the selected Protocol and Provider checkouts' canonical package-build actions and stage their wheels with a selected fixture. | | `provider/start` | Run the fixture as a separate non-root systemd service. | @@ -199,7 +207,7 @@ scenario its own job and clean container, so one failure cannot contaminate a later scenario. Use one of the `smoke`, `capability`, `lifecycle`, `health`, `triggers`, -`adversarial`, or `destructive` tiers. A scenario action must always invoke cleanup with +`operations`, `adversarial`, or `destructive` tiers. A scenario action must always invoke cleanup with `always()` so failure evidence is retained. Adding a normal scenario requires one action and one thin workflow job; it must not duplicate RPM installation, systemd readiness, diff --git a/docs/configuration.md b/docs/configuration.md index a80417b..961c6a0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -32,10 +32,14 @@ health_interval_seconds = 15 reconnect_initial_delay_seconds = 1 reconnect_max_delay_seconds = 30 trigger_max_in_flight_publications = 32 +operation_admission_timeout_seconds = 5 +operation_admission_attempts = 2 +operation_admission_retry_initial_delay_seconds = 1 [journal] backend = "sqlite" maximum_pending_trigger_publications = 10000 +maximum_unresolved_operation_admissions = 10000 [logging] level = "info" @@ -109,6 +113,9 @@ does not check whether the referenced file exists during `check-config`. | `reconnect_initial_delay_seconds` | Integer | `1` | Initial bounded-backoff delay after a failed Provider connection. Must be greater than zero. | | `reconnect_max_delay_seconds` | Integer | `30` | Maximum reconnect delay. Must be greater than zero and greater than or equal to the initial delay. | | `trigger_max_in_flight_publications` | Integer | `32` | Requested SFP Trigger Publication receive window. Must be between 1 and the Protocol-owned maximum of 1,024. | +| `operation_admission_timeout_seconds` | Integer | `5` | Finite deadline for one SFP Operation Admission RPC attempt. Must be greater than zero. | +| `operation_admission_attempts` | Integer | `2` | Maximum attempts for one immutable admission identity. Must be between 1 and 5. | +| `operation_admission_retry_initial_delay_seconds` | Integer | `1` | Initial delay for a retryable admission failure. Must be greater than zero. | The RPM creates the Provider discovery directory through `systemd-tmpfiles`. Provider processes publish their endpoint descriptors and Unix domain sockets @@ -120,6 +127,7 @@ there; the Agent does not use this directory for Provider-specific data. | --- | --- | --- | --- | | `backend` | String | `sqlite` | Selects the durable journal implementation. `sqlite` selects the bundled SQLite implementation and is the only backend shipped in schema version 1. | | `maximum_pending_trigger_publications` | Integer | `10000` | Maximum accepted trigger publications retained for later SF-AOP forwarding. Must be greater than zero. When full, new publications receive a retryable capacity rejection. | +| `maximum_unresolved_operation_admissions` | Integer | `10000` | Maximum durable operation intents whose remote admission outcome is not yet proven. Must be greater than zero. When full, new intents are rejected before Provider contact. | The journal database is stored at the package-owned path `/var/lib/stateforge-agent/journal/agent.db`. diff --git a/docs/index.md b/docs/index.md index 09eae73..1a9ac37 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,10 @@ The current runtime can: operations without converting that evidence into authorization; and - consume manifest-declared Provider triggers, validate their payloads, and durably assume bounded local delivery responsibility before acknowledging - them. + them; and +- admit manifest-declared operations only after current capability evidence, + persist their immutable identities before transport, and recover uncertain + responses without dispatching duplicate work. The runtime reports a Provider as connected only after both session setup and manifest validation succeed. Operational eligibility is stricter: the exact @@ -51,3 +54,8 @@ Health reports `SERVING`. The Agent validates each event against the canonical P3 contract and accepted manifest schema, then commits its idempotency record before returning an acknowledgement. Accepted events remain pending in the Agent journal until the later SF-AOP forwarding boundary is implemented. + +Operation Admission is similarly fenced to the exact available session, +manifest, declaration, and capability evidence. A lost response is retried +with the same durable operation identity; the Provider resolves the existing +record rather than running the operation twice. diff --git a/docs/sfp-operation-admission.md b/docs/sfp-operation-admission.md new file mode 100644 index 0000000..49d73ee --- /dev/null +++ b/docs/sfp-operation-admission.md @@ -0,0 +1,107 @@ +# Operation Admission + +The Agent uses SFP Operation Admission to submit one authorized, immutable +operation intent to an eligible Provider. The Provider either creates the +operation record and dispatches the handler, or resolves the same durable +identity to the record it created earlier. + +Operation Admission does not define authorization policy or operation +progress. The caller authorizes the request before entering this boundary; +later SFP procedures report execution state, results, and cancellation. + +```mermaid +sequenceDiagram + participant C as Agent caller + participant J as Durable Agent journal + participant A as Agent admission client + participant P as Provider + + C->>A: admit_operation(request) + A->>A: Check manifest, schema, profiles, and A3 evidence + A->>J: Persist immutable intent and fingerprint + J-->>A: Durable operation identity + A->>P: StartOperation(ADMIT_OR_RESOLVE) + P->>P: Atomically record before dispatch + P-->>A: ACCEPTED_NEW + A->>J: Persist accepted record + A-->>C: OperationAdmissionResult + + Note over A,P: If the response is lost, the Agent retries the same identity + A->>P: StartOperation(same invocation and fingerprint) + P-->>A: RESOLVED_EXISTING +``` + +## Agent API + +`ProviderSessionManager.admit_operation` accepts an +`OperationAdmissionRequest` containing: + +- the Provider installation identity; +- the manifest-declared operation identifier; +- the exact JSON payload and execution context; +- a finite execution deadline; and +- optional required implementation profiles. + +The Agent allocates the operation UUID, binds it to the immutable request +fingerprint, and persists that intent before contacting the Provider. The +caller never supplies or reuses an arbitrary operation UUID. + +Admission requires the exact current Provider session to be available. The +operation must be declared in the accepted manifest, its payload must satisfy +the declared Draft 2020-12 schema, every requested implementation profile must +be claimed by the operation, and the current Capability Assessment must cover +the same operation and execution context with an admissible support state. + +## Uncertain outcomes + +A transport failure does not prove that the Provider rejected the operation. +The Provider may have committed and dispatched it before the response was +lost. The Agent therefore preserves the original identity and immutable +content instead of creating another operation. + +| Situation | Agent behavior | +| --- | --- | +| Response is lost on the current session | Retry the exact request with the same operation UUID and fingerprint. | +| Provider already recorded the request | Accept `RESOLVED_EXISTING`; do not create a second operation. | +| Reconnection creates a new session | Use resolve-only recovery for the existing identity. | +| Provider cannot prove the retained record | Keep the outcome unresolved; never silently admit a replacement operation. | +| Same identity is bound to changed content | Reject the local conflict before transport. | + +The durable record survives Agent restart. Until later lifecycle procedures +complete, an unresolved admission remains explicit Agent responsibility. + +## Bounds and safety + +Three Provider settings bound transport retry behavior: + +- `providers.operation_admission_timeout_seconds` sets the finite deadline for + each RPC attempt; +- `providers.operation_admission_attempts` limits attempts to between 1 and 5; +- `providers.operation_admission_retry_initial_delay_seconds` sets the initial + bounded retry delay. + +`journal.maximum_unresolved_operation_admissions` limits durable intents whose +remote outcome is not yet proven. When the limit is reached, the Agent rejects +new intents before contacting a Provider. + +Canonical message sizes, fingerprints, response correlation, error details, +and status mappings come from `stateforge-protocol`. Payload and execution +context values are retained only in the durable journal and wire request; they +are not copied into logs, exception text, or CI evidence. + +## Installed verification + +The installed-system scenario builds the Agent RPM and exact Protocol and +Provider wheels, then starts the packaged Agent and a real Provider Python +process as separate systemd services. The Provider commits and dispatches one +operation, deliberately drops the first successful response, and returns the +existing record when the Agent retries. + +The scenario proves that: + +- malformed or undeclared requests are rejected locally; +- one immutable request is sent twice with the same operation UUID and digest; +- the real Provider handler runs exactly once; +- the Agent receives `RESOLVED_EXISTING` after the simulated response loss; +- the accepted record survives an Agent service restart; and +- evidence contains identifiers and hashes, never operation payloads. diff --git a/mkdocs.yml b/mkdocs.yml index fb4f83a..bedff0b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - Running the Agent: running.md - Configuration: configuration.md - Capability Assessment: sfp-capability-assessment.md + - Operation Admission: sfp-operation-admission.md - Trigger Publication: sfp-trigger-publication.md - RPM CI/CD: ci-cd.md - Internal reference: diff --git a/packaging/rpm/SOURCES/stateforge-agent.toml b/packaging/rpm/SOURCES/stateforge-agent.toml index 179db21..1393d7e 100644 --- a/packaging/rpm/SOURCES/stateforge-agent.toml +++ b/packaging/rpm/SOURCES/stateforge-agent.toml @@ -19,10 +19,14 @@ health_interval_seconds = 15 reconnect_initial_delay_seconds = 1 reconnect_max_delay_seconds = 30 trigger_max_in_flight_publications = 32 +operation_admission_timeout_seconds = 5 +operation_admission_attempts = 2 +operation_admission_retry_initial_delay_seconds = 1 [journal] backend = "sqlite" maximum_pending_trigger_publications = 10000 +maximum_unresolved_operation_admissions = 10000 [logging] level = "info" diff --git a/pyproject.toml b/pyproject.toml index 9f92450..6d94c25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ classifiers = [ dependencies = [ "grpcio>=1.83", "grpcio-health-checking>=1.83", + "grpcio-status>=1.83", "jsonschema>=4.25,<5", "stateforge-protocol>=0.1.0,<0.2.0", "watchfiles>=1.2,<2", @@ -60,6 +61,7 @@ dev = [ "ruff>=0.12", "types-grpcio", "types-grpcio-health-checking", + "types-grpcio-status", "types-jsonschema", "types-protobuf", ] diff --git a/src/stateforge_agent/_config/loading.py b/src/stateforge_agent/_config/loading.py index c1637ae..2ff4b9e 100644 --- a/src/stateforge_agent/_config/loading.py +++ b/src/stateforge_agent/_config/loading.py @@ -40,6 +40,7 @@ Provenance: TypeAlias = dict[ConfigPath, Path] _SUPPORTED_SCHEMA_VERSION = 1 +_MAX_OPERATION_ADMISSION_ATTEMPTS = 5 _TOP_LEVEL_KEYS = { "schema_version", "control_plane", @@ -335,18 +336,32 @@ def _validate_configuration( "reconnect_initial_delay_seconds", "reconnect_max_delay_seconds", "trigger_max_in_flight_publications", + "operation_admission_timeout_seconds", + "operation_admission_attempts", + "operation_admission_retry_initial_delay_seconds", }, required={ "health_interval_seconds", "reconnect_initial_delay_seconds", "reconnect_max_delay_seconds", "trigger_max_in_flight_publications", + "operation_admission_timeout_seconds", + "operation_admission_attempts", + "operation_admission_retry_initial_delay_seconds", }, ) validator.table( ("journal",), - allowed={"backend", "maximum_pending_trigger_publications"}, - required={"backend", "maximum_pending_trigger_publications"}, + allowed={ + "backend", + "maximum_pending_trigger_publications", + "maximum_unresolved_operation_admissions", + }, + required={ + "backend", + "maximum_pending_trigger_publications", + "maximum_unresolved_operation_admissions", + }, ) validator.table( ("logging",), @@ -386,6 +401,20 @@ def _validate_configuration( trigger_max_in_flight = validator.positive_integer( ("providers", "trigger_max_in_flight_publications") ) + operation_admission_timeout = validator.positive_integer( + ("providers", "operation_admission_timeout_seconds") + ) + operation_admission_attempts = validator.positive_integer( + ("providers", "operation_admission_attempts") + ) + if operation_admission_attempts > _MAX_OPERATION_ADMISSION_ATTEMPTS: + validator.add_issue( + ("providers", "operation_admission_attempts"), + "must not exceed 5", + ) + operation_admission_retry_initial_delay = validator.positive_integer( + ("providers", "operation_admission_retry_initial_delay_seconds") + ) if ( trigger_max_in_flight > trigger_publication_contract.MAX_IN_FLIGHT_PUBLICATIONS @@ -414,6 +443,9 @@ def _validate_configuration( maximum_pending_trigger_publications = validator.positive_integer( ("journal", "maximum_pending_trigger_publications") ) + maximum_unresolved_operation_admissions = validator.positive_integer( + ("journal", "maximum_unresolved_operation_admissions") + ) log_level = validator.choice( ("logging", "level"), @@ -438,12 +470,20 @@ def _validate_configuration( reconnect_initial_delay_seconds=reconnect_initial_delay, reconnect_max_delay_seconds=reconnect_max_delay, trigger_max_in_flight_publications=trigger_max_in_flight, + operation_admission_timeout_seconds=(operation_admission_timeout), + operation_admission_attempts=operation_admission_attempts, + operation_admission_retry_initial_delay_seconds=( + operation_admission_retry_initial_delay + ), ), journal=JournalConfiguration( backend=journal_backend, maximum_pending_trigger_publications=( maximum_pending_trigger_publications ), + maximum_unresolved_operation_admissions=( + maximum_unresolved_operation_admissions + ), ), logging=LoggingConfiguration( level=cast(LogLevel, log_level), diff --git a/src/stateforge_agent/_config/models.py b/src/stateforge_agent/_config/models.py index a551929..ffa1fa6 100644 --- a/src/stateforge_agent/_config/models.py +++ b/src/stateforge_agent/_config/models.py @@ -34,6 +34,9 @@ class ProviderConfiguration: reconnect_initial_delay_seconds: int reconnect_max_delay_seconds: int trigger_max_in_flight_publications: int + operation_admission_timeout_seconds: int + operation_admission_attempts: int + operation_admission_retry_initial_delay_seconds: int @dataclass(frozen=True, slots=True) @@ -42,6 +45,7 @@ class JournalConfiguration: backend: JournalBackendName maximum_pending_trigger_publications: int + maximum_unresolved_operation_admissions: int @dataclass(frozen=True, slots=True) diff --git a/src/stateforge_agent/_journal/base.py b/src/stateforge_agent/_journal/base.py index 1042c3a..1dd6e59 100644 --- a/src/stateforge_agent/_journal/base.py +++ b/src/stateforge_agent/_journal/base.py @@ -8,6 +8,9 @@ if TYPE_CHECKING: from stateforge_agent._config.models import JournalConfiguration + from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, + ) from stateforge_agent._sfp._services.trigger_publication.store import ( TriggerPublicationStore, ) @@ -47,3 +50,11 @@ def trigger_publications( maximum_pending_publications: int, ) -> TriggerPublicationStore: """Create the backend's durable Trigger Publication store.""" + + @abstractmethod + def operation_admissions( + self, + *, + maximum_unresolved_attempts: int, + ) -> OperationAdmissionStore: + """Create the backend's durable Operation Admission store.""" diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py index 0d01728..64f1787 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite.py @@ -10,9 +10,15 @@ from typing import TYPE_CHECKING, Self, TypeVar from stateforge_agent._journal.base import Journal, JournalError +from stateforge_agent._journal.sqlite_operation_admissions import ( + SqliteOperationAdmissionStore, +) from stateforge_agent._journal.sqlite_trigger_publications import ( SqliteTriggerPublicationStore, ) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, +) from stateforge_agent._sfp._services.trigger_publication.store import ( TriggerPublicationStore, ) @@ -106,6 +112,17 @@ def trigger_publications( maximum_pending_publications=maximum_pending_publications, ) + def operation_admissions( + self, + *, + maximum_unresolved_attempts: int, + ) -> OperationAdmissionStore: + """Create the SQLite Operation Admission adapter.""" + return SqliteOperationAdmissionStore( + self, + maximum_unresolved_attempts=maximum_unresolved_attempts, + ) + def _prepare(self) -> None: """Prepare the directory and durable SQLite mode synchronously.""" self._path.parent.mkdir( diff --git a/src/stateforge_agent/_journal/sqlite_operation_admissions.py b/src/stateforge_agent/_journal/sqlite_operation_admissions.py new file mode 100644 index 0000000..a007391 --- /dev/null +++ b/src/stateforge_agent/_journal/sqlite_operation_admissions.py @@ -0,0 +1,473 @@ +"""Persist Agent-owned Operation Admission intent in SQLite.""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from typing import Protocol, TypeVar +from uuid import UUID + +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, +) + +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionDisposition, + OperationAdmissionResult, + OperationRejection, +) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, + OperationAttempt, + OperationAttemptState, + OperationIntentDecision, + OperationIntentResult, +) + +_SCHEMA_COMPONENT = "operation_admission" +_SCHEMA_VERSION = 1 +_Result = TypeVar("_Result") + + +class _SqliteTransactions(Protocol): + """Expose only the transaction boundary needed by this adapter.""" + + async def transaction( + self, + operation: Callable[[sqlite3.Connection], _Result], + ) -> _Result: + """Run one serialized SQLite transaction.""" + + +class SqliteOperationAdmissionStore(OperationAdmissionStore): + """Own durable immutable operation intent and resolution state.""" + + def __init__( + self, + journal: _SqliteTransactions, + *, + maximum_unresolved_attempts: int, + ) -> None: + """Configure one bounded operation-intent store.""" + if maximum_unresolved_attempts <= 0: + raise ValueError("maximum_unresolved_attempts must be positive") + self._journal = journal + self._maximum_unresolved_attempts = maximum_unresolved_attempts + + async def initialize(self) -> None: + """Create the first admission schema or verify its version.""" + await self._journal.transaction(self._initialize) + + async def create_intent( + self, + provider_instance_id: UUID, + invocation: operation_admission_pb2.OperationInvocation, + request_fingerprint: str, + ) -> OperationIntentResult: + """Atomically insert or classify one immutable invocation.""" + serialized = invocation.SerializeToString(deterministic=True) + now = datetime.now(UTC).isoformat() + + def create(connection: sqlite3.Connection) -> OperationIntentResult: + existing = self._select( + connection, + provider_instance_id, + UUID(invocation.operation_id), + ) + if existing is not None: + decision = ( + OperationIntentDecision.EXISTING + if existing.request_fingerprint == request_fingerprint + else OperationIntentDecision.CONFLICT + ) + return OperationIntentResult(decision, existing) + if self._unresolved_count(connection) >= ( + self._maximum_unresolved_attempts + ): + return OperationIntentResult( + OperationIntentDecision.CAPACITY_EXHAUSTED, + None, + ) + connection.execute( + """ + INSERT INTO operation_admission_attempts ( + provider_instance_id, + operation_id, + invocation, + request_fingerprint, + state, + created_at, + updated_at, + attempt_count + ) VALUES (?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + str(provider_instance_id), + invocation.operation_id, + serialized, + request_fingerprint, + OperationAttemptState.UNRESOLVED.value, + now, + now, + ), + ) + inserted = self._select( + connection, + provider_instance_id, + UUID(invocation.operation_id), + ) + if inserted is None: + raise sqlite3.DatabaseError("Operation intent was not stored") + return OperationIntentResult( + OperationIntentDecision.CREATED, + inserted, + ) + + return await self._journal.transaction(create) + + async def get( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAttempt | None: + """Return one durable attempt when known.""" + return await self._journal.transaction( + lambda connection: self._select( + connection, + provider_instance_id, + operation_id, + ) + ) + + async def record_transport_attempt( + self, + provider_instance_id: UUID, + operation_id: UUID, + *, + endpoint_generation: int, + session_id: str, + ) -> OperationAttempt: + """Record one exact session attempt and reopen retryable rejection.""" + now = datetime.now(UTC).isoformat() + + def record(connection: sqlite3.Connection) -> OperationAttempt: + updated = connection.execute( + """ + UPDATE operation_admission_attempts + SET state = ?, updated_at = ?, + attempt_count = attempt_count + 1, + last_endpoint_generation = ?, last_session_id = ?, + rejection_reason = NULL, rejection_retryable = NULL, + rejection_retry_after_seconds = NULL, + rejection_explanation = NULL, + rejection_grpc_status = NULL + WHERE provider_instance_id = ? AND operation_id = ? + """, + ( + OperationAttemptState.UNRESOLVED.value, + now, + endpoint_generation, + session_id, + str(provider_instance_id), + str(operation_id), + ), + ) + if updated.rowcount != 1: + raise sqlite3.DatabaseError("Operation intent is absent") + attempt = self._select( + connection, + provider_instance_id, + operation_id, + ) + if attempt is None: + raise sqlite3.DatabaseError("Operation intent is absent") + return attempt + + return await self._journal.transaction(record) + + async def record_accepted( + self, + result: OperationAdmissionResult, + ) -> OperationAttempt: + """Resolve one durable attempt from a validated success response.""" + now = datetime.now(UTC).isoformat() + + def accept(connection: sqlite3.Connection) -> OperationAttempt: + updated = connection.execute( + """ + UPDATE operation_admission_attempts + SET state = ?, updated_at = ?, accepted_disposition = ?, + accepted_endpoint_generation = ?, + accepted_session_id = ?, accepted_manifest_revision = ?, + admitted_at = ?, record_expires_at = ?, + admitted_endpoint_generation = ? + WHERE provider_instance_id = ? AND operation_id = ? + AND request_fingerprint = ? + """, + ( + OperationAttemptState.ACCEPTED.value, + now, + result.disposition.value, + result.endpoint_generation, + result.session_id, + result.manifest_revision, + result.admitted_at.isoformat(), + result.record_expires_at.isoformat(), + result.admitted_endpoint_generation, + str(result.provider_instance_id), + str(result.operation_id), + result.request_fingerprint, + ), + ) + if updated.rowcount != 1: + raise sqlite3.DatabaseError( + "Operation success does not match durable intent" + ) + attempt = self._select( + connection, + result.provider_instance_id, + result.operation_id, + ) + if attempt is None: + raise sqlite3.DatabaseError("Operation intent is absent") + return attempt + + return await self._journal.transaction(accept) + + async def record_rejected( + self, + provider_instance_id: UUID, + operation_id: UUID, + rejection: OperationRejection, + ) -> OperationAttempt: + """Record one canonical rejection without storing request content.""" + now = datetime.now(UTC).isoformat() + retry_after = ( + rejection.retry_after.total_seconds() + if rejection.retry_after is not None + else None + ) + + def reject(connection: sqlite3.Connection) -> OperationAttempt: + updated = connection.execute( + """ + UPDATE operation_admission_attempts + SET state = ?, updated_at = ?, rejection_reason = ?, + rejection_retryable = ?, + rejection_retry_after_seconds = ?, + rejection_explanation = ?, rejection_grpc_status = ? + WHERE provider_instance_id = ? AND operation_id = ? + """, + ( + OperationAttemptState.REJECTED.value, + now, + rejection.reason, + int(rejection.retryable), + retry_after, + rejection.explanation, + rejection.grpc_status, + str(provider_instance_id), + str(operation_id), + ), + ) + if updated.rowcount != 1: + raise sqlite3.DatabaseError("Operation intent is absent") + attempt = self._select( + connection, + provider_instance_id, + operation_id, + ) + if attempt is None: + raise sqlite3.DatabaseError("Operation intent is absent") + return attempt + + return await self._journal.transaction(reject) + + async def unresolved(self) -> tuple[OperationAttempt, ...]: + """Return unresolved attempts without exposing serialized content.""" + + def read( + connection: sqlite3.Connection, + ) -> tuple[OperationAttempt, ...]: + rows = connection.execute( + f"{self._SELECT} WHERE state = ? ORDER BY created_at", + (OperationAttemptState.UNRESOLVED.value,), + ).fetchall() + return tuple(self._attempt_from_row(row) for row in rows) + + return await self._journal.transaction(read) + + _SELECT = """ + SELECT provider_instance_id, operation_id, invocation, + request_fingerprint, state, created_at, updated_at, + attempt_count, last_endpoint_generation, last_session_id, + accepted_disposition, accepted_endpoint_generation, + accepted_session_id, accepted_manifest_revision, admitted_at, + record_expires_at, admitted_endpoint_generation, + rejection_reason, rejection_retryable, + rejection_retry_after_seconds, rejection_explanation, + rejection_grpc_status + FROM operation_admission_attempts + """ + + def _select( + self, + connection: sqlite3.Connection, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAttempt | None: + query = ( + f"{self._SELECT} WHERE provider_instance_id = ? " + "AND operation_id = ?" + ) + row = connection.execute( + query, + (str(provider_instance_id), str(operation_id)), + ).fetchone() + return self._attempt_from_row(row) if row is not None else None + + @staticmethod + def _attempt_from_row(row: tuple[object, ...]) -> OperationAttempt: + invocation = operation_admission_pb2.OperationInvocation.FromString( + _bytes(row[2]) + ) + state = OperationAttemptState(_string(row[4])) + result = None + if state is OperationAttemptState.ACCEPTED: + result = OperationAdmissionResult( + provider_instance_id=UUID(_string(row[0])), + endpoint_generation=_integer(row[11]), + session_id=_string(row[12]), + manifest_revision=_string(row[13]), + operation_id=UUID(_string(row[1])), + request_fingerprint=_string(row[3]), + disposition=OperationAdmissionDisposition(_string(row[10])), + admitted_at=datetime.fromisoformat(_string(row[14])), + record_expires_at=datetime.fromisoformat(_string(row[15])), + admitted_endpoint_generation=_integer(row[16]), + ) + rejection = None + if state is OperationAttemptState.REJECTED: + rejection = OperationRejection( + operation_id=UUID(_string(row[1])), + reason=_string(row[17]), + retryable=bool(_integer(row[18])), + retry_after=( + timedelta(seconds=_number(row[19])) + if row[19] is not None + else None + ), + explanation=_string(row[20]), + grpc_status=_string(row[21]), + ) + return OperationAttempt( + provider_instance_id=UUID(_string(row[0])), + operation_id=UUID(_string(row[1])), + invocation=invocation, + request_fingerprint=_string(row[3]), + state=state, + created_at=datetime.fromisoformat(_string(row[5])), + updated_at=datetime.fromisoformat(_string(row[6])), + attempt_count=_integer(row[7]), + last_endpoint_generation=( + _integer(row[8]) if row[8] is not None else None + ), + last_session_id=(_string(row[9]) if row[9] is not None else None), + result=result, + rejection=rejection, + ) + + @staticmethod + def _unresolved_count(connection: sqlite3.Connection) -> int: + row = connection.execute( + """ + SELECT COUNT(*) FROM operation_admission_attempts + WHERE state = ? + """, + (OperationAttemptState.UNRESOLVED.value,), + ).fetchone() + if row is None: + raise sqlite3.DatabaseError("Operation intent count is absent") + return _integer(row[0]) + + @staticmethod + def _initialize(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS journal_schemas ( + component TEXT PRIMARY KEY, + version INTEGER NOT NULL CHECK (version > 0) + ) WITHOUT ROWID + """ + ) + row = connection.execute( + "SELECT version FROM journal_schemas WHERE component = ?", + (_SCHEMA_COMPONENT,), + ).fetchone() + if row is not None and row[0] != _SCHEMA_VERSION: + raise sqlite3.DatabaseError( + "Unsupported Operation Admission journal schema" + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS operation_admission_attempts ( + provider_instance_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + invocation BLOB NOT NULL, + request_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK ( + state IN ('unresolved', 'accepted', 'rejected') + ), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + attempt_count INTEGER NOT NULL CHECK (attempt_count >= 0), + last_endpoint_generation INTEGER, + last_session_id TEXT, + accepted_disposition TEXT, + accepted_endpoint_generation INTEGER, + accepted_session_id TEXT, + accepted_manifest_revision TEXT, + admitted_at TEXT, + record_expires_at TEXT, + admitted_endpoint_generation INTEGER, + rejection_reason TEXT, + rejection_retryable INTEGER, + rejection_retry_after_seconds REAL, + rejection_explanation TEXT, + rejection_grpc_status TEXT, + PRIMARY KEY (provider_instance_id, operation_id) + ) WITHOUT ROWID + """ + ) + connection.execute( + """ + INSERT OR IGNORE INTO journal_schemas (component, version) + VALUES (?, ?) + """, + (_SCHEMA_COMPONENT, _SCHEMA_VERSION), + ) + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise sqlite3.DatabaseError("Operation journal text is invalid") + return value + + +def _integer(value: object) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise sqlite3.DatabaseError("Operation journal integer is invalid") + return value + + +def _number(value: object) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise sqlite3.DatabaseError("Operation journal number is invalid") + return float(value) + + +def _bytes(value: object) -> bytes: + if not isinstance(value, bytes): + raise sqlite3.DatabaseError("Operation journal bytes are invalid") + return value diff --git a/src/stateforge_agent/_runtime/daemon.py b/src/stateforge_agent/_runtime/daemon.py index f424800..16b1e81 100644 --- a/src/stateforge_agent/_runtime/daemon.py +++ b/src/stateforge_agent/_runtime/daemon.py @@ -39,6 +39,9 @@ ProviderDirectoryEventKind, ProviderDirectoryMonitor, ) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCoordinator, +) from stateforge_agent._sfp._services.trigger_publication.coordinator import ( TriggerPublicationCoordinator, ) @@ -80,9 +83,15 @@ async def _run_agent( configuration.journal.maximum_pending_trigger_publications ), ) + operation_store = journal.operation_admissions( + maximum_unresolved_attempts=( + configuration.journal.maximum_unresolved_operation_admissions + ), + ) try: await journal.open() await trigger_store.initialize() + await operation_store.initialize() except JournalError: _LOGGER.exception("Agent journal initialization failed") for installed_signal in installed_signals: @@ -119,6 +128,16 @@ def reload_provider_descriptor( configuration.providers.trigger_max_in_flight_publications ), ), + operation_admission=OperationAdmissionCoordinator( + operation_store, + timeout_seconds=( + configuration.providers.operation_admission_timeout_seconds + ), + attempts=configuration.providers.operation_admission_attempts, + retry_initial_delay_seconds=( + configuration.providers.operation_admission_retry_initial_delay_seconds + ), + ), ) try: diff --git a/src/stateforge_agent/_sfp/__init__.py b/src/stateforge_agent/_sfp/__init__.py index 6cbb087..85d1674 100644 --- a/src/stateforge_agent/_sfp/__init__.py +++ b/src/stateforge_agent/_sfp/__init__.py @@ -31,6 +31,27 @@ SfpCapabilityAssessmentTransportError, SfpCapabilityAssessmentUnavailableError, ) +from stateforge_agent._sfp._services.operation_admission import ( + OperationAdmissionDisposition, + OperationAdmissionRequest, + OperationAdmissionResult, + OperationRejection, + SfpOperationAdmissionDeadlineError, + SfpOperationAdmissionError, + SfpOperationAdmissionLimitError, + SfpOperationAdmissionProtocolError, + SfpOperationAdmissionRejectedError, + SfpOperationAdmissionRequestError, + SfpOperationAdmissionStaleError, + SfpOperationAdmissionTransportError, + SfpOperationAdmissionUnavailableError, +) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCapacityError, + OperationAdmissionConflictError, + OperationAdmissionPreconditionError, + OperationAdmissionUnresolvedError, +) from stateforge_agent._sfp.version import SfpVersion __all__ = [ @@ -41,6 +62,14 @@ "CapabilityRequirement", "CapabilityRequirementState", "CapabilitySupportState", + "OperationAdmissionCapacityError", + "OperationAdmissionConflictError", + "OperationAdmissionDisposition", + "OperationAdmissionPreconditionError", + "OperationAdmissionRequest", + "OperationAdmissionResult", + "OperationAdmissionUnresolvedError", + "OperationRejection", "ProviderAvailabilityState", "ProviderAvailabilityStatus", "ProviderConnectionState", @@ -59,5 +88,14 @@ "SfpConnectionError", "SfpIncompatibleError", "SfpManifestError", + "SfpOperationAdmissionDeadlineError", + "SfpOperationAdmissionError", + "SfpOperationAdmissionLimitError", + "SfpOperationAdmissionProtocolError", + "SfpOperationAdmissionRejectedError", + "SfpOperationAdmissionRequestError", + "SfpOperationAdmissionStaleError", + "SfpOperationAdmissionTransportError", + "SfpOperationAdmissionUnavailableError", "SfpVersion", ] diff --git a/src/stateforge_agent/_sfp/_runtime/sessions.py b/src/stateforge_agent/_sfp/_runtime/sessions.py index 1a26318..e996430 100644 --- a/src/stateforge_agent/_sfp/_runtime/sessions.py +++ b/src/stateforge_agent/_sfp/_runtime/sessions.py @@ -12,6 +12,7 @@ from uuid import UUID import grpc +from google.protobuf.json_format import MessageToDict from grpc_health.v1 import health_pb2 from stateforge_protocol.sfp import primitives @@ -50,6 +51,18 @@ from stateforge_agent._sfp._services.manifest.declaration import ( ManifestDeclaration, ) +from stateforge_agent._sfp._services.operation_admission.client import ( + SfpOperationAdmissionStaleError, +) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCoordinator, + OperationAdmissionLease, + OperationAdmissionPreconditionError, +) +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionRequest, + OperationAdmissionResult, +) from stateforge_agent._sfp._services.trigger_publication.client import ( SfpTriggerPublicationError, SfpTriggerPublicationStaleError, @@ -181,6 +194,7 @@ def __init__( CapabilityAssessmentRequester | None ) = None, trigger_publication: TriggerPublicationCoordinator | None = None, + operation_admission: OperationAdmissionCoordinator | None = None, ) -> None: """Create an empty Provider session registry.""" if not isinstance(agent_runtime_id, UUID): @@ -246,6 +260,7 @@ def __init__( client=capability_assessment_client, ) self._trigger_publication = trigger_publication + self._operation_admission = operation_admission self._providers: dict[UUID, _ProviderRuntimeState] = {} self._closed = False self._lock = asyncio.Lock() @@ -308,6 +323,113 @@ async def cached_capability_assessment( assert_current=self._assert_capability_assessment_is_current, ) + async def admit_operation( + self, + request: OperationAdmissionRequest, + ) -> OperationAdmissionResult: + """Preflight and submit one newly authorized generic operation.""" + coordinator = self._operation_admission + if coordinator is None: + raise OperationAdmissionPreconditionError( + "Operation Admission is not configured" + ) + assessment = await self.assess_capabilities( + request.provider_instance_id, + (request.operation_identifier,), + execution_context=request.execution_context, + ) + lease = await self._operation_admission_lease( + request.provider_instance_id + ) + return await coordinator.admit( + lease, + request, + assessment, + assert_current=self._assert_operation_admission_is_current, + ) + + async def resolve_operation( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAdmissionResult: + """Resolve one durable uncertain identity through a current session.""" + coordinator = self._operation_admission + if coordinator is None: + raise OperationAdmissionPreconditionError( + "Operation Admission is not configured" + ) + attempt = await coordinator.attempt(provider_instance_id, operation_id) + if attempt is None: + raise OperationAdmissionPreconditionError( + "Operation Admission identity is not durable" + ) + execution_context = MessageToDict(attempt.invocation.execution_context) + assessment = await self.assess_capabilities( + provider_instance_id, + (attempt.invocation.operation_identifier,), + execution_context=execution_context, + ) + lease = await self._operation_admission_lease(provider_instance_id) + return await coordinator.submit( + lease, + operation_id, + assessment, + assert_current=self._assert_operation_admission_is_current, + ) + + async def _operation_admission_lease( + self, + provider_instance_id: UUID, + ) -> OperationAdmissionLease: + """Snapshot current operation declarations and availability fences.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if provider is None or provider.active_session is None: + raise OperationAdmissionPreconditionError( + "Provider has no active SFP session" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise OperationAdmissionPreconditionError( + "Provider is not available for Operation Admission" + ) + session = provider.active_session + return OperationAdmissionLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + manifest_revision=session.manifest.key.revision, + availability_generation=provider.availability_generation, + operation_declarations=( + session.manifest.operation_declarations + ), + channel=session._channel, + session_token=session, + ) + + async def _assert_operation_admission_is_current( + self, + lease: OperationAdmissionLease, + ) -> None: + """Fence one admission attempt to current session and readiness.""" + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + provider is None + or provider.active_session is not lease.session_token + or provider.availability_generation + != lease.availability_generation + ): + raise SfpOperationAdmissionStaleError( + "Operation Admission session is no longer current" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise SfpOperationAdmissionStaleError( + "Provider is no longer available for Operation Admission" + ) + async def _capability_assessment_lease( self, provider_instance_id: UUID, @@ -655,6 +777,8 @@ async def close(self) -> None: """Close every active Provider channel.""" async with self._lock: self._closed = True + if self._operation_admission is not None: + self._operation_admission.close() sessions = tuple( provider.active_session for provider in self._providers.values() diff --git a/src/stateforge_agent/_sfp/_services/manifest/declaration.py b/src/stateforge_agent/_sfp/_services/manifest/declaration.py index ea0dc50..ae9901f 100644 --- a/src/stateforge_agent/_sfp/_services/manifest/declaration.py +++ b/src/stateforge_agent/_sfp/_services/manifest/declaration.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Any from uuid import UUID @@ -26,6 +28,14 @@ class ManifestCacheKey: revision: str +@dataclass(frozen=True, slots=True) +class ManifestOperationDeclaration: + """Expose the manifest facts needed to preflight one operation.""" + + input_schema: Mapping[str, Any] + implements: frozenset[str] + + @dataclass(frozen=True, slots=True) class ManifestDeclaration: """Retain one validated normalized declaration as canonical JSON.""" @@ -64,6 +74,34 @@ def operation_identifiers(self) -> frozenset[str]: raise RuntimeError("Validated manifest cache entry is corrupt") return frozenset(operations) + @property + def operation_declarations( + self, + ) -> Mapping[str, ManifestOperationDeclaration]: + """Return immutable preflight declarations keyed by identifier.""" + operations = self.as_dict().get("operations") + if not isinstance(operations, dict): + raise RuntimeError("Validated manifest cache entry is corrupt") + declarations: dict[str, ManifestOperationDeclaration] = {} + for identifier, declaration in operations.items(): + if not isinstance(identifier, str) or not isinstance( + declaration, dict + ): + raise RuntimeError("Validated manifest cache entry is corrupt") + input_schema = declaration.get("inputSchema") + implements = declaration.get("implements") + if ( + not isinstance(input_schema, dict) + or not isinstance(implements, list) + or any(not isinstance(item, str) for item in implements) + ): + raise RuntimeError("Validated manifest cache entry is corrupt") + declarations[identifier] = ManifestOperationDeclaration( + input_schema=MappingProxyType(input_schema), + implements=frozenset(implements), + ) + return MappingProxyType(declarations) + @property def trigger_schemas(self) -> dict[str, dict[str, Any]]: """Return payload schemas keyed by declared trigger identifier.""" diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/__init__.py b/src/stateforge_agent/_sfp/_services/operation_admission/__init__.py new file mode 100644 index 0000000..6af95b6 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_admission/__init__.py @@ -0,0 +1,35 @@ +"""Agent-side SFP Operation Admission coordination.""" + +from stateforge_agent._sfp._services.operation_admission.client import ( + SfpOperationAdmissionDeadlineError, + SfpOperationAdmissionError, + SfpOperationAdmissionLimitError, + SfpOperationAdmissionProtocolError, + SfpOperationAdmissionRejectedError, + SfpOperationAdmissionRequestError, + SfpOperationAdmissionStaleError, + SfpOperationAdmissionTransportError, + SfpOperationAdmissionUnavailableError, +) +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionDisposition, + OperationAdmissionRequest, + OperationAdmissionResult, + OperationRejection, +) + +__all__ = [ + "OperationAdmissionDisposition", + "OperationAdmissionRequest", + "OperationAdmissionResult", + "OperationRejection", + "SfpOperationAdmissionDeadlineError", + "SfpOperationAdmissionError", + "SfpOperationAdmissionLimitError", + "SfpOperationAdmissionProtocolError", + "SfpOperationAdmissionRejectedError", + "SfpOperationAdmissionRequestError", + "SfpOperationAdmissionStaleError", + "SfpOperationAdmissionTransportError", + "SfpOperationAdmissionUnavailableError", +] diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/client.py b/src/stateforge_agent/_sfp/_services/operation_admission/client.py new file mode 100644 index 0000000..046c248 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_admission/client.py @@ -0,0 +1,279 @@ +"""Call SFP Operation Admission and normalize validated outcomes.""" + +from __future__ import annotations + +from collections.abc import Awaitable +from dataclasses import dataclass +from datetime import UTC +from typing import Never, Protocol, cast +from uuid import UUID + +import grpc +from google.protobuf.message import DecodeError +from grpc_status import rpc_status +from stateforge_protocol.sfp.operation_admission.v1 import ( + contract, + operation_admission_pb2, + validation, +) + +from stateforge_agent._configuration_validation import require_finite_positive +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionDisposition, + OperationAdmissionResult, + OperationRejection, +) + + +class SfpOperationAdmissionError(RuntimeError): + """Report a sanitized Agent-side admission failure.""" + + +class SfpOperationAdmissionRequestError(SfpOperationAdmissionError): + """Report an invalid locally constructed request.""" + + +class SfpOperationAdmissionProtocolError(SfpOperationAdmissionError): + """Report malformed, foreign, or inconsistent Provider data.""" + + +class SfpOperationAdmissionUnavailableError(SfpOperationAdmissionError): + """Report transport loss without an admission conclusion.""" + + +class SfpOperationAdmissionDeadlineError(SfpOperationAdmissionError): + """Report expiry of one finite transport deadline.""" + + +class SfpOperationAdmissionStaleError(SfpOperationAdmissionError): + """Report a stale session, generation, or declaration fence.""" + + +class SfpOperationAdmissionLimitError(SfpOperationAdmissionError): + """Report a request or response exceeding canonical limits.""" + + +class SfpOperationAdmissionTransportError(SfpOperationAdmissionError): + """Report a non-retryable unstructured transport failure.""" + + +class SfpOperationAdmissionRejectedError(SfpOperationAdmissionError): + """Carry one validated canonical Provider rejection.""" + + def __init__(self, rejection: OperationRejection) -> None: + super().__init__( + "Operation Admission rejected: " + f"{rejection.reason} ({rejection.grpc_status})" + ) + self.rejection = rejection + + +class OperationAdmissionRequester(Protocol): + """Describe one finite canonical Operation Admission RPC attempt.""" + + async def start( + self, + channel: grpc.aio.Channel, + request: operation_admission_pb2.StartOperationRequest, + ) -> OperationAdmissionResult: + """Return one validated admission result.""" + + +class _OperationUnaryCall(Protocol): + def __call__( + self, + request: operation_admission_pb2.StartOperationRequest, + *, + timeout: float, + ) -> Awaitable[_DecodedAdmissionResponse]: + """Issue one unary admission request.""" + + +@dataclass(frozen=True, slots=True) +class _DecodedAdmissionResponse: + response: operation_admission_pb2.StartOperationResponse | None = None + malformed: bool = False + limit_exceeded: bool = False + + +_SERVICE_DESCRIPTOR = operation_admission_pb2.DESCRIPTOR.services_by_name[ + "OperationAdmissionService" +] +_START_DESCRIPTOR = _SERVICE_DESCRIPTOR.methods_by_name["StartOperation"] +_START_METHOD_PATH = ( + f"/{_SERVICE_DESCRIPTOR.full_name}/{_START_DESCRIPTOR.name}" +) + + +class OperationAdmissionClient: + """Perform one bounded Operation Admission transport attempt.""" + + def __init__(self, *, timeout_seconds: float = 5.0) -> None: + self._timeout_seconds = require_finite_positive( + timeout_seconds, + name="Operation Admission timeout", + ) + + async def start( + self, + channel: grpc.aio.Channel, + request: operation_admission_pb2.StartOperationRequest, + ) -> OperationAdmissionResult: + """Validate, call, correlate, and normalize one response.""" + validate_request(request) + call: _OperationUnaryCall = channel.unary_unary( + _START_METHOD_PATH, + request_serializer=( + operation_admission_pb2.StartOperationRequest.SerializeToString + ), + response_deserializer=_decode_response, + ) + try: + decoded = await call(request, timeout=self._timeout_seconds) + except grpc.aio.AioRpcError as error: + rejection = _canonical_rejection( + error, + expected_operation_id=request.invocation.operation_id, + ) + if rejection is not None: + raise SfpOperationAdmissionRejectedError(rejection) from None + _raise_transport_error(error.code()) + + if decoded.limit_exceeded: + raise SfpOperationAdmissionLimitError( + "Operation Admission response limit exceeded" + ) + if decoded.malformed or decoded.response is None: + raise SfpOperationAdmissionProtocolError( + "Operation Admission response protobuf is malformed" + ) + response = decoded.response + try: + validation.validate_admission_exchange(request, response) + except validation.OperationAdmissionValidationError as error: + raise SfpOperationAdmissionProtocolError( + "Operation Admission response failed canonical validation" + ) from error + return _normalize(response) + + +def validate_request( + request: operation_admission_pb2.StartOperationRequest, +) -> None: + """Delegate context-independent request rules to Protocol.""" + if request.ByteSize() > contract.MAX_REQUEST_BYTES: + raise SfpOperationAdmissionLimitError( + "Operation Admission request limit exceeded" + ) + try: + validation.validate_start_operation_request(request) + except validation.OperationAdmissionValidationError as error: + raise SfpOperationAdmissionRequestError( + "Operation Admission request failed canonical validation" + ) from error + + +def _decode_response(payload: bytes) -> _DecodedAdmissionResponse: + if len(payload) > contract.MAX_RESPONSE_BYTES: + return _DecodedAdmissionResponse(limit_exceeded=True) + try: + response = operation_admission_pb2.StartOperationResponse.FromString( + payload + ) + except DecodeError: + return _DecodedAdmissionResponse(malformed=True) + return _DecodedAdmissionResponse(response=response) + + +def _canonical_rejection( + error: grpc.aio.AioRpcError, + *, + expected_operation_id: str, +) -> OperationRejection | None: + status = rpc_status.from_call(cast(grpc.Call, error)) + if status is None or len(status.details) != 1: + return None + detail = operation_admission_pb2.OperationAdmissionErrorDetail() + if not status.details[0].Unpack(detail): + return None + try: + validation.validate_admission_error_detail(detail) + except validation.OperationAdmissionValidationError as validation_error: + raise SfpOperationAdmissionProtocolError( + "Operation Admission rejection failed canonical validation" + ) from validation_error + if detail.operation_id and detail.operation_id != expected_operation_id: + raise SfpOperationAdmissionProtocolError( + "Operation Admission rejection identity mismatch" + ) + operation_id = UUID(detail.operation_id) if detail.operation_id else None + retry_after = ( + detail.retry_after.ToTimedelta() + if detail.HasField("retry_after") + else None + ) + return OperationRejection( + operation_id=operation_id, + reason=( + operation_admission_pb2.AdmissionErrorReason.Name(detail.reason) + ), + retryable=detail.retryable, + retry_after=retry_after, + explanation=detail.explanation, + grpc_status=error.code().name, + ) + + +def _raise_transport_error(code: grpc.StatusCode) -> Never: + if code is grpc.StatusCode.UNAVAILABLE: + raise SfpOperationAdmissionUnavailableError( + "Operation Admission is temporarily unavailable" + ) + if code is grpc.StatusCode.DEADLINE_EXCEEDED: + raise SfpOperationAdmissionDeadlineError( + "Operation Admission transport deadline expired" + ) + if code is grpc.StatusCode.FAILED_PRECONDITION: + raise SfpOperationAdmissionStaleError( + "Operation Admission session or declaration is stale" + ) + if code is grpc.StatusCode.RESOURCE_EXHAUSTED: + raise SfpOperationAdmissionLimitError( + "Operation Admission transport limit exceeded" + ) + if code is grpc.StatusCode.INVALID_ARGUMENT: + raise SfpOperationAdmissionRequestError( + "Provider rejected the Operation Admission request" + ) + raise SfpOperationAdmissionTransportError( + f"Operation Admission transport failed: {code.name}" + ) + + +def _normalize( + response: operation_admission_pb2.StartOperationResponse, +) -> OperationAdmissionResult: + disposition = { + operation_admission_pb2.ADMISSION_DISPOSITION_ACCEPTED_NEW: ( + OperationAdmissionDisposition.ACCEPTED_NEW + ), + operation_admission_pb2.ADMISSION_DISPOSITION_RESOLVED_EXISTING: ( + OperationAdmissionDisposition.RESOLVED_EXISTING + ), + }[response.disposition] + return OperationAdmissionResult( + provider_instance_id=UUID(response.provider_instance_id), + endpoint_generation=response.endpoint_generation, + session_id=response.session_id, + manifest_revision=response.manifest_revision, + operation_id=UUID(response.record.operation_id), + request_fingerprint=response.record.request_fingerprint, + disposition=disposition, + admitted_at=response.record.admitted_at.ToDatetime(tzinfo=UTC), + record_expires_at=response.record.record_expires_at.ToDatetime( + tzinfo=UTC + ), + admitted_endpoint_generation=( + response.record.admitted_endpoint_generation + ), + ) diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py new file mode 100644 index 0000000..4b7db97 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py @@ -0,0 +1,582 @@ +"""Coordinate Agent-side Operation Admission and uncertainty recovery.""" + +from __future__ import annotations + +import asyncio +import hashlib +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import grpc +from google.protobuf import struct_pb2, timestamp_pb2 +from google.protobuf.json_format import ParseDict, ParseError +from jsonschema import Draft202012Validator +from jsonschema.exceptions import ValidationError +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, + validation, +) + +from stateforge_agent._configuration_validation import ( + require_bounded_integer, + require_finite_non_negative, + require_finite_positive, +) +from stateforge_agent._journal.base import JournalError +from stateforge_agent._sfp._services.capability_assessment.models import ( + CapabilityAssessment, + CapabilityRequirementState, + CapabilitySupportState, +) +from stateforge_agent._sfp._services.manifest.declaration import ( + ManifestOperationDeclaration, +) +from stateforge_agent._sfp._services.operation_admission.client import ( + OperationAdmissionClient, + OperationAdmissionRequester, + SfpOperationAdmissionDeadlineError, + SfpOperationAdmissionError, + SfpOperationAdmissionProtocolError, + SfpOperationAdmissionRejectedError, + SfpOperationAdmissionStaleError, + SfpOperationAdmissionUnavailableError, + validate_request, +) +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionRequest, + OperationAdmissionResult, + OperationRejection, +) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, + OperationAttempt, + OperationIntentDecision, +) + +_MAX_ATTEMPTS = 5 +_LOCK_STRIPES = 64 +_UNRESOLVED_REASONS = { + "ADMISSION_ERROR_REASON_OPERATION_RECORD_UNAVAILABLE", + "ADMISSION_ERROR_REASON_OPERATION_RECORD_EXPIRED", +} + + +class OperationAdmissionLocalError(RuntimeError): + """Report a sanitized Agent-owned admission decision.""" + + +class OperationAdmissionPreconditionError(OperationAdmissionLocalError): + """Report that local declaration or evidence preflight failed.""" + + +class OperationAdmissionConflictError(OperationAdmissionLocalError): + """Report changed immutable content under one operation identity.""" + + +class OperationAdmissionCapacityError(OperationAdmissionLocalError): + """Report that durable unresolved-attempt capacity is exhausted.""" + + +class OperationAdmissionUnresolvedError(OperationAdmissionLocalError): + """Report an identity whose remote admission outcome is not proven.""" + + def __init__(self, operation_id: UUID, reason: str) -> None: + super().__init__( + f"Operation Admission outcome is unresolved: {operation_id}" + ) + self.operation_id = operation_id + self.reason = reason + + +@dataclass(frozen=True, slots=True) +class OperationAdmissionLease: + """Expose only current session facts needed by Operation Admission.""" + + provider_instance_id: UUID + endpoint_generation: int + session_id: str + manifest_revision: str + availability_generation: int + operation_declarations: Mapping[str, ManifestOperationDeclaration] + channel: grpc.aio.Channel = field(repr=False, compare=False) + session_token: object = field(repr=False, compare=False) + + +OperationAdmissionCurrentnessCheck = Callable[ + [OperationAdmissionLease], Awaitable[None] +] + + +class OperationAdmissionCoordinator: + """Own immutable intent, P4 transport, retry, and durable resolution.""" + + def __init__( + self, + store: OperationAdmissionStore, + *, + timeout_seconds: float = 5.0, + attempts: int = 2, + retry_initial_delay_seconds: float = 0.25, + client: OperationAdmissionRequester | None = None, + ) -> None: + """Configure bounded admission attempts over one durable store.""" + timeout_seconds = require_finite_positive( + timeout_seconds, + name="Operation Admission timeout", + ) + self._attempts = require_bounded_integer( + attempts, + name="Operation Admission attempts", + minimum=1, + maximum=_MAX_ATTEMPTS, + ) + self._retry_initial_delay_seconds = require_finite_non_negative( + retry_initial_delay_seconds, + name="Operation Admission retry delay", + ) + self._store = store + self._client = client or OperationAdmissionClient( + timeout_seconds=timeout_seconds + ) + self._locks = tuple(asyncio.Lock() for _ in range(_LOCK_STRIPES)) + self._closed = False + + async def admit( + self, + lease: OperationAdmissionLease, + request: OperationAdmissionRequest, + assessment: CapabilityAssessment, + *, + assert_current: OperationAdmissionCurrentnessCheck, + ) -> OperationAdmissionResult: + """Prepare and submit one newly authorized immutable attempt.""" + operation_id = await self.prepare( + lease, + request, + assessment, + assert_current=assert_current, + ) + return await self.submit( + lease, + operation_id, + assessment, + assert_current=assert_current, + ) + + async def prepare( + self, + lease: OperationAdmissionLease, + request: OperationAdmissionRequest, + assessment: CapabilityAssessment, + *, + assert_current: OperationAdmissionCurrentnessCheck, + ) -> UUID: + """Validate and durably create one identity before transport.""" + if self._closed: + raise OperationAdmissionPreconditionError( + "Operation Admission coordinator is closed" + ) + await assert_current(lease) + invocation = _invocation(lease, request) + _preflight(lease, invocation, request, assessment) + validate_request( + _request( + lease, + invocation, + operation_admission_pb2.ADMISSION_MODE_ADMIT_OR_RESOLVE, + ) + ) + fingerprint = validation.compute_request_fingerprint(invocation) + try: + stored = await self._store.create_intent( + lease.provider_instance_id, + invocation, + fingerprint, + ) + except JournalError as error: + raise OperationAdmissionCapacityError( + "Operation Admission intent could not be persisted" + ) from error + if stored.decision is OperationIntentDecision.CONFLICT: + raise OperationAdmissionConflictError( + "Operation identity is bound to different immutable content" + ) + if stored.decision is OperationIntentDecision.CAPACITY_EXHAUSTED: + raise OperationAdmissionCapacityError( + "Operation Admission unresolved-attempt capacity is exhausted" + ) + if stored.attempt is None: + raise RuntimeError("Operation Admission store omitted its attempt") + await assert_current(lease) + return stored.attempt.operation_id + + async def submit( + self, + lease: OperationAdmissionLease, + operation_id: UUID, + assessment: CapabilityAssessment, + *, + assert_current: OperationAdmissionCurrentnessCheck, + ) -> OperationAdmissionResult: + """Submit or resolve one already durable operation identity.""" + key = (lease.provider_instance_id, operation_id) + lock = self._locks[hash(key) % len(self._locks)] + async with lock: + return await self._submit_locked( + lease, + operation_id, + assessment, + assert_current=assert_current, + ) + + async def attempt( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAttempt | None: + """Return durable state for recovery or operator inspection.""" + return await self._store.get(provider_instance_id, operation_id) + + def close(self) -> None: + """Reject new preparations while preserving durable attempts.""" + self._closed = True + + async def _submit_locked( + self, + lease: OperationAdmissionLease, + operation_id: UUID, + assessment: CapabilityAssessment, + *, + assert_current: OperationAdmissionCurrentnessCheck, + ) -> OperationAdmissionResult: + attempt = await self._pending_attempt(lease, operation_id) + if isinstance(attempt, OperationAdmissionResult): + return attempt + _preflight_existing(lease, attempt, assessment) + mode = _mode(lease, attempt) + for attempt_number in range(self._attempts): + await assert_current(lease) + await self._require_live_deadline(lease, attempt) + await self._store.record_transport_attempt( + lease.provider_instance_id, + operation_id, + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + ) + request = _request(lease, attempt.invocation, mode) + try: + result = await self._client.start(lease.channel, request) + except SfpOperationAdmissionRejectedError as error: + await self._handle_rejection( + lease, + attempt, + attempt_number, + error, + ) + continue + except ( + SfpOperationAdmissionUnavailableError, + SfpOperationAdmissionDeadlineError, + ) as error: + await self._retry_transport_error( + attempt, + attempt_number, + error, + ) + continue + except ( + SfpOperationAdmissionProtocolError, + SfpOperationAdmissionStaleError, + ) as error: + raise OperationAdmissionUnresolvedError( + operation_id, + type(error).__name__, + ) from error + except SfpOperationAdmissionError: + raise + await assert_current(lease) + try: + await self._store.record_accepted(result) + except JournalError as error: + raise OperationAdmissionUnresolvedError( + operation_id, + "accepted response could not be persisted", + ) from error + return result + raise RuntimeError("Operation Admission retry loop exhausted") + + async def _pending_attempt( + self, + lease: OperationAdmissionLease, + operation_id: UUID, + ) -> OperationAttempt | OperationAdmissionResult: + attempt = await self._store.get( + lease.provider_instance_id, operation_id + ) + if attempt is None: + raise OperationAdmissionPreconditionError( + "Operation Admission identity is not durable" + ) + if attempt.result is not None: + return attempt.result + if attempt.rejection is not None and not attempt.rejection.retryable: + raise SfpOperationAdmissionRejectedError(attempt.rejection) + return attempt + + async def _require_live_deadline( + self, + lease: OperationAdmissionLease, + attempt: OperationAttempt, + ) -> None: + if _deadline(attempt.invocation) > datetime.now(UTC): + return + rejection = OperationRejection( + operation_id=attempt.operation_id, + reason="ADMISSION_ERROR_REASON_OPERATION_DEADLINE_EXPIRED", + retryable=False, + retry_after=None, + explanation="Operation execution deadline has elapsed.", + grpc_status=grpc.StatusCode.DEADLINE_EXCEEDED.name, + ) + await self._store.record_rejected( + lease.provider_instance_id, + attempt.operation_id, + rejection, + ) + raise SfpOperationAdmissionRejectedError(rejection) + + async def _handle_rejection( + self, + lease: OperationAdmissionLease, + attempt: OperationAttempt, + attempt_number: int, + error: SfpOperationAdmissionRejectedError, + ) -> None: + rejection = error.rejection + if rejection.reason in _UNRESOLVED_REASONS: + raise OperationAdmissionUnresolvedError( + attempt.operation_id, + rejection.reason, + ) from error + await self._store.record_rejected( + lease.provider_instance_id, + attempt.operation_id, + rejection, + ) + if not rejection.retryable or attempt_number + 1 >= self._attempts: + raise error + await _retry_delay( + rejection, + fallback=self._retry_initial_delay_seconds * (2**attempt_number), + deadline=_deadline(attempt.invocation), + ) + + async def _retry_transport_error( + self, + attempt: OperationAttempt, + attempt_number: int, + error: SfpOperationAdmissionError, + ) -> None: + if attempt_number + 1 >= self._attempts: + raise OperationAdmissionUnresolvedError( + attempt.operation_id, + type(error).__name__, + ) from error + await asyncio.sleep( + self._retry_initial_delay_seconds * (2**attempt_number) + ) + + +def _invocation( + lease: OperationAdmissionLease, + request: OperationAdmissionRequest, +) -> operation_admission_pb2.OperationInvocation: + if request.provider_instance_id != lease.provider_instance_id: + raise OperationAdmissionPreconditionError( + "Operation Admission Provider identity is not current" + ) + if request.execution_deadline.tzinfo is None: + raise OperationAdmissionPreconditionError( + "Operation execution deadline must be timezone-aware" + ) + payload = _structure(request.payload, name="operation payload") + execution_context = _structure( + request.execution_context, + name="operation execution context", + ) + deadline = timestamp_pb2.Timestamp() + deadline.FromDatetime(request.execution_deadline.astimezone(UTC)) + return operation_admission_pb2.OperationInvocation( + operation_id=str(uuid4()), + operation_identifier=request.operation_identifier, + payload=payload, + execution_deadline=deadline, + contract=operation_admission_pb2.OperationContractContext( + manifest_revision=lease.manifest_revision, + required_implementation_roles=sorted( + request.required_implementation_roles + ), + ), + execution_context=execution_context, + ) + + +def _structure( + value: Mapping[str, object], + *, + name: str, +) -> struct_pb2.Struct: + message = struct_pb2.Struct() + try: + ParseDict(dict(value), message) + except (ParseError, TypeError, ValueError) as error: + raise OperationAdmissionPreconditionError( + f"{name} is not valid JSON" + ) from error + return message + + +def _preflight( + lease: OperationAdmissionLease, + invocation: operation_admission_pb2.OperationInvocation, + request: OperationAdmissionRequest, + assessment: CapabilityAssessment, +) -> None: + declaration = lease.operation_declarations.get( + invocation.operation_identifier + ) + if declaration is None: + raise OperationAdmissionPreconditionError( + "Operation is not declared by the accepted manifest" + ) + try: + Draft202012Validator(declaration.input_schema).validate( + dict(request.payload) + ) + except ValidationError as error: + raise OperationAdmissionPreconditionError( + "Operation payload does not match its declared schema" + ) from error + if not set(request.required_implementation_roles).issubset( + declaration.implements + ): + raise OperationAdmissionPreconditionError( + "Operation requires an undeclared implementation profile" + ) + _validate_assessment(lease, invocation, assessment) + + +def _preflight_existing( + lease: OperationAdmissionLease, + attempt: OperationAttempt, + assessment: CapabilityAssessment, +) -> None: + invocation = attempt.invocation + if invocation.contract.manifest_revision != lease.manifest_revision: + raise OperationAdmissionPreconditionError( + "Operation Admission manifest revision is no longer current" + ) + declaration = lease.operation_declarations.get( + invocation.operation_identifier + ) + if declaration is None or not set( + invocation.contract.required_implementation_roles + ).issubset(declaration.implements): + raise OperationAdmissionPreconditionError( + "Operation Admission declaration is no longer compatible" + ) + _validate_assessment(lease, invocation, assessment) + + +def _validate_assessment( + lease: OperationAdmissionLease, + invocation: operation_admission_pb2.OperationInvocation, + assessment: CapabilityAssessment, +) -> None: + scope = assessment.scope + context_digest = hashlib.sha256( + invocation.execution_context.SerializeToString(deterministic=True) + ).digest() + if ( + scope.provider_instance_id != lease.provider_instance_id + or scope.endpoint_generation != lease.endpoint_generation + or scope.session_id != lease.session_id + or scope.manifest_revision != lease.manifest_revision + or scope.availability_generation != lease.availability_generation + or scope.operation_identifiers != (invocation.operation_identifier,) + or scope.execution_context_sha256 != context_digest + ): + raise OperationAdmissionPreconditionError( + "Capability Assessment evidence does not match admission scope" + ) + if len(assessment.findings) != 1: + raise OperationAdmissionPreconditionError( + "Capability Assessment did not return exactly one finding" + ) + finding = assessment.findings[0] + if finding.operation_identifier != invocation.operation_identifier: + raise OperationAdmissionPreconditionError( + "Capability Assessment finding identity is inconsistent" + ) + if finding.support_state not in { + CapabilitySupportState.SUPPORTED, + CapabilitySupportState.SUPPORTED_WITH_REQUIREMENTS, + } or any( + requirement.state is not CapabilityRequirementState.SATISFIED + for requirement in finding.requirements + ): + raise OperationAdmissionPreconditionError( + "Capability Assessment does not support this operation" + ) + + +def _request( + lease: OperationAdmissionLease, + invocation: operation_admission_pb2.OperationInvocation, + mode: operation_admission_pb2.AdmissionMode.ValueType, +) -> operation_admission_pb2.StartOperationRequest: + return operation_admission_pb2.StartOperationRequest( + session_binding=operation_admission_pb2.OperationSessionBinding( + expected_provider_instance_id=str(lease.provider_instance_id), + expected_endpoint_generation=lease.endpoint_generation, + expected_session_id=lease.session_id, + ), + invocation=invocation, + admission_mode=mode, + ) + + +def _mode( + lease: OperationAdmissionLease, + attempt: OperationAttempt, +) -> operation_admission_pb2.AdmissionMode.ValueType: + if ( + attempt.last_endpoint_generation is not None + and attempt.last_endpoint_generation != lease.endpoint_generation + ): + return operation_admission_pb2.ADMISSION_MODE_RESOLVE_ONLY + return operation_admission_pb2.ADMISSION_MODE_ADMIT_OR_RESOLVE + + +def _deadline( + invocation: operation_admission_pb2.OperationInvocation, +) -> datetime: + return invocation.execution_deadline.ToDatetime(tzinfo=UTC) + + +async def _retry_delay( + rejection: OperationRejection, + *, + fallback: float, + deadline: datetime, +) -> None: + delay = ( + rejection.retry_after.total_seconds() + if rejection.retry_after is not None + else fallback + ) + if datetime.now(UTC).timestamp() + delay >= deadline.timestamp(): + return + await asyncio.sleep(delay) diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/models.py b/src/stateforge_agent/_sfp/_services/operation_admission/models.py new file mode 100644 index 0000000..d24a3ef --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_admission/models.py @@ -0,0 +1,66 @@ +"""Define immutable Agent-normalized Operation Admission data.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta +from enum import StrEnum +from uuid import UUID + +JsonValue = ( + str + | int + | float + | bool + | None + | list["JsonValue"] + | dict[str, "JsonValue"] +) + + +class OperationAdmissionDisposition(StrEnum): + """Describe how the Provider resolved one immutable identity.""" + + ACCEPTED_NEW = "accepted_new" + RESOLVED_EXISTING = "resolved_existing" + + +@dataclass(frozen=True, slots=True) +class OperationAdmissionRequest: + """Describe one Agent-authorized generic operation attempt.""" + + provider_instance_id: UUID + operation_identifier: str + payload: Mapping[str, JsonValue] + execution_deadline: datetime + execution_context: Mapping[str, JsonValue] + required_implementation_roles: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class OperationAdmissionResult: + """Expose one validated Provider admission record.""" + + provider_instance_id: UUID + endpoint_generation: int + session_id: str + manifest_revision: str + operation_id: UUID + request_fingerprint: str + disposition: OperationAdmissionDisposition + admitted_at: datetime + record_expires_at: datetime + admitted_endpoint_generation: int + + +@dataclass(frozen=True, slots=True) +class OperationRejection: + """Preserve one canonical sanitized Provider rejection.""" + + operation_id: UUID | None + reason: str + retryable: bool + retry_after: timedelta | None + explanation: str + grpc_status: str diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/store.py b/src/stateforge_agent/_sfp/_services/operation_admission/store.py new file mode 100644 index 0000000..aae2727 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_admission/store.py @@ -0,0 +1,120 @@ +"""Define storage-neutral durable Operation Admission intent.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from enum import StrEnum +from uuid import UUID + +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, +) + +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionResult, + OperationRejection, +) + + +class OperationAttemptState(StrEnum): + """Describe the Agent's durable knowledge of one admission attempt.""" + + UNRESOLVED = "unresolved" + ACCEPTED = "accepted" + REJECTED = "rejected" + + +class OperationIntentDecision(StrEnum): + """Classify an atomic immutable-intent creation request.""" + + CREATED = "created" + EXISTING = "existing" + CONFLICT = "conflict" + CAPACITY_EXHAUSTED = "capacity_exhausted" + + +@dataclass(frozen=True, slots=True) +class OperationAttempt: + """Represent one immutable operation identity and current knowledge.""" + + provider_instance_id: UUID + operation_id: UUID + invocation: operation_admission_pb2.OperationInvocation = field( + repr=False, + compare=False, + ) + request_fingerprint: str + state: OperationAttemptState + created_at: datetime + updated_at: datetime + attempt_count: int + last_endpoint_generation: int | None + last_session_id: str | None + result: OperationAdmissionResult | None = None + rejection: OperationRejection | None = None + + +@dataclass(frozen=True, slots=True) +class OperationIntentResult: + """Return an atomic creation decision and current durable attempt.""" + + decision: OperationIntentDecision + attempt: OperationAttempt | None + + +class OperationAdmissionStore(ABC): + """Persist immutable Agent intent independently of a storage vendor.""" + + @abstractmethod + async def initialize(self) -> None: + """Create or verify this procedure's journal schema.""" + + @abstractmethod + async def create_intent( + self, + provider_instance_id: UUID, + invocation: operation_admission_pb2.OperationInvocation, + request_fingerprint: str, + ) -> OperationIntentResult: + """Atomically create or classify one immutable operation identity.""" + + @abstractmethod + async def get( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAttempt | None: + """Return one durable attempt when known.""" + + @abstractmethod + async def record_transport_attempt( + self, + provider_instance_id: UUID, + operation_id: UUID, + *, + endpoint_generation: int, + session_id: str, + ) -> OperationAttempt: + """Record the exact session used for one transport attempt.""" + + @abstractmethod + async def record_accepted( + self, + result: OperationAdmissionResult, + ) -> OperationAttempt: + """Resolve one durable attempt from a validated success response.""" + + @abstractmethod + async def record_rejected( + self, + provider_instance_id: UUID, + operation_id: UUID, + rejection: OperationRejection, + ) -> OperationAttempt: + """Record one canonical rejection without claiming execution.""" + + @abstractmethod + async def unresolved(self) -> tuple[OperationAttempt, ...]: + """Return unresolved attempts available for restart recovery.""" diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 9ca3285..1ef266b 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -26,10 +26,14 @@ reconnect_initial_delay_seconds = 1 reconnect_max_delay_seconds = 30 trigger_max_in_flight_publications = 32 +operation_admission_timeout_seconds = 5 +operation_admission_attempts = 2 +operation_admission_retry_initial_delay_seconds = 1 [journal] backend = "sqlite" maximum_pending_trigger_publications = 10000 +maximum_unresolved_operation_admissions = 10000 [logging] level = "info" @@ -43,6 +47,7 @@ _INVALID_INPUT_EXIT_CODE = 2 _TRIGGER_RECEIVE_WINDOW = 32 _TRIGGER_PENDING_CAPACITY = 10_000 +_OPERATION_ADMISSION_ATTEMPTS = 2 def test_process_exit_codes_are_stable() -> None: @@ -75,6 +80,14 @@ def test_loads_complete_configuration(tmp_path: Path) -> None: loaded.config.journal.maximum_pending_trigger_publications == _TRIGGER_PENDING_CAPACITY ) + assert ( + loaded.config.providers.operation_admission_attempts + == _OPERATION_ADMISSION_ATTEMPTS + ) + assert ( + loaded.config.journal.maximum_unresolved_operation_admissions + == _TRIGGER_PENDING_CAPACITY + ) assert loaded.sources == (config_path,) assert loaded.fingerprint.startswith("sha256:") diff --git a/tests/test_daemon_provider_events.py b/tests/test_daemon_provider_events.py index c8491d7..68fc293 100644 --- a/tests/test_daemon_provider_events.py +++ b/tests/test_daemon_provider_events.py @@ -81,10 +81,14 @@ def _configuration(directory: Path) -> AgentConfiguration: reconnect_initial_delay_seconds=1, reconnect_max_delay_seconds=30, trigger_max_in_flight_publications=32, + operation_admission_timeout_seconds=5, + operation_admission_attempts=2, + operation_admission_retry_initial_delay_seconds=1, ), journal=JournalConfiguration( backend="sqlite", maximum_pending_trigger_publications=10000, + maximum_unresolved_operation_admissions=10000, ), logging=LoggingConfiguration( level="info", diff --git a/tests/test_journal_registry.py b/tests/test_journal_registry.py index fa16fca..9437342 100644 --- a/tests/test_journal_registry.py +++ b/tests/test_journal_registry.py @@ -12,6 +12,9 @@ SqliteJournal, create_journal, ) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, +) from stateforge_agent._sfp._services.trigger_publication.store import ( TriggerPublicationStore, ) @@ -49,11 +52,20 @@ def trigger_publications( del maximum_pending_publications raise NotImplementedError + def operation_admissions( + self, + *, + maximum_unresolved_attempts: int, + ) -> OperationAdmissionStore: + del maximum_unresolved_attempts + raise NotImplementedError + def _configuration(backend: str = "sqlite") -> JournalConfiguration: return JournalConfiguration( backend=backend, maximum_pending_trigger_publications=10, + maximum_unresolved_operation_admissions=10, ) diff --git a/tests/test_operation_admission.py b/tests/test_operation_admission.py new file mode 100644 index 0000000..7842ea3 --- /dev/null +++ b/tests/test_operation_admission.py @@ -0,0 +1,687 @@ +"""Agent-side Operation Admission preflight and recovery tests.""" + +from __future__ import annotations + +import asyncio +import hashlib +from collections.abc import Awaitable, Callable +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import cast +from uuid import UUID, uuid4 + +import grpc +import pytest +from google.protobuf import any_pb2, duration_pb2 +from google.protobuf.json_format import ParseDict +from google.protobuf.timestamp_pb2 import Timestamp +from google.rpc import status_pb2 +from grpc_status import rpc_status +from stateforge_protocol.sfp.operation_admission.v1 import ( + contract, + operation_admission_pb2, + validation, +) + +from stateforge_agent._journal import JournalError, SqliteJournal +from stateforge_agent._sfp._services.capability_assessment.models import ( + CapabilityAssessment, + CapabilityAssessmentScope, + CapabilityFinding, + CapabilitySupportState, +) +from stateforge_agent._sfp._services.manifest.declaration import ( + ManifestOperationDeclaration, +) +from stateforge_agent._sfp._services.operation_admission.client import ( + OperationAdmissionClient, + OperationAdmissionRequester, + SfpOperationAdmissionLimitError, + SfpOperationAdmissionProtocolError, + SfpOperationAdmissionRejectedError, + SfpOperationAdmissionStaleError, + SfpOperationAdmissionUnavailableError, + _decode_response, +) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCapacityError, + OperationAdmissionCoordinator, + OperationAdmissionLease, + OperationAdmissionPreconditionError, + OperationAdmissionUnresolvedError, +) +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionDisposition, + OperationAdmissionRequest, + OperationAdmissionResult, +) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, +) + +_PROVIDER_ID = UUID("7f7db7d8-68b4-4f71-9769-065b5c4f941c") +_MANIFEST_REVISION = "sha256:" + ("a" * 64) +_OPERATION = "stateforge.test/restart@1" +_PROFILE = "stateforge.test.profile.v1.safe_restart" +_LOST_RESPONSE_CALL_COUNT = 2 +_OVERSIZED_RESPONSE_LENGTH = contract.MAX_RESPONSE_BYTES + 1 +_INPUT_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"unit": {"type": "string"}}, + "required": ["unit"], + "additionalProperties": False, +} + + +class _ScriptedClient(OperationAdmissionRequester): + """Return deterministic results while retaining exact wire requests.""" + + def __init__( + self, + steps: list[ + OperationAdmissionDisposition + | SfpOperationAdmissionUnavailableError + ], + ) -> None: + self.steps = steps + self.requests: list[operation_admission_pb2.StartOperationRequest] = [] + + async def start( + self, + channel: grpc.aio.Channel, + request: operation_admission_pb2.StartOperationRequest, + ) -> OperationAdmissionResult: + del channel + self.requests.append(request) + step = self.steps.pop(0) + if isinstance(step, Exception): + raise step + now = datetime.now(UTC) + return OperationAdmissionResult( + provider_instance_id=UUID( + request.session_binding.expected_provider_instance_id + ), + endpoint_generation=( + request.session_binding.expected_endpoint_generation + ), + session_id=request.session_binding.expected_session_id, + manifest_revision=request.invocation.contract.manifest_revision, + operation_id=UUID(request.invocation.operation_id), + request_fingerprint=validation.compute_request_fingerprint( + request.invocation + ), + disposition=step, + admitted_at=now, + record_expires_at=now + timedelta(hours=1), + admitted_endpoint_generation=( + request.session_binding.expected_endpoint_generation + ), + ) + + +def _lease(*, generation: int = 1) -> OperationAdmissionLease: + return OperationAdmissionLease( + provider_instance_id=_PROVIDER_ID, + endpoint_generation=generation, + session_id=f"session-{generation}", + manifest_revision=_MANIFEST_REVISION, + availability_generation=generation, + operation_declarations={ + _OPERATION: ManifestOperationDeclaration( + input_schema=_INPUT_SCHEMA, + implements=frozenset({_PROFILE}), + ) + }, + channel=cast(grpc.aio.Channel, object()), + session_token=object(), + ) + + +def _request( + *, + unit: str = "nginx.service", + roles: tuple[str, ...] = (_PROFILE,), +) -> OperationAdmissionRequest: + return OperationAdmissionRequest( + provider_instance_id=_PROVIDER_ID, + operation_identifier=_OPERATION, + payload={"unit": unit}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=5), + execution_context={"requester": "test"}, + required_implementation_roles=roles, + ) + + +def _assessment(lease: OperationAdmissionLease) -> CapabilityAssessment: + context = operation_admission_pb2.OperationInvocation() + ParseDict({"requester": "test"}, context.execution_context) + return CapabilityAssessment( + scope=CapabilityAssessmentScope( + provider_instance_id=lease.provider_instance_id, + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + availability_generation=lease.availability_generation, + operation_identifiers=(_OPERATION,), + execution_context_sha256=hashlib.sha256( + context.execution_context.SerializeToString(deterministic=True) + ).digest(), + ), + request_id=uuid4(), + assessment_id=uuid4(), + assessed_at=datetime.now(UTC), + findings=( + CapabilityFinding( + operation_identifier=_OPERATION, + support_state=CapabilitySupportState.SUPPORTED, + requirements=(), + reasons=(), + constraints=(), + ), + ), + wire_size_bytes=1, + ) + + +async def _current(lease: OperationAdmissionLease) -> None: + del lease + + +def _wire_request() -> operation_admission_pb2.StartOperationRequest: + invocation = operation_admission_pb2.OperationInvocation( + operation_id=str(uuid4()), + operation_identifier=_OPERATION, + execution_deadline=Timestamp( + seconds=int((datetime.now(UTC) + timedelta(minutes=5)).timestamp()) + ), + contract=operation_admission_pb2.OperationContractContext( + manifest_revision=_MANIFEST_REVISION, + required_implementation_roles=[_PROFILE], + ), + ) + ParseDict({"unit": "nginx.service"}, invocation.payload) + ParseDict({"requester": "test"}, invocation.execution_context) + return operation_admission_pb2.StartOperationRequest( + session_binding=operation_admission_pb2.OperationSessionBinding( + expected_provider_instance_id=str(_PROVIDER_ID), + expected_endpoint_generation=1, + expected_session_id="session-1", + ), + invocation=invocation, + admission_mode=( + operation_admission_pb2.ADMISSION_MODE_ADMIT_OR_RESOLVE + ), + ) + + +def _wire_response( + request: operation_admission_pb2.StartOperationRequest, +) -> operation_admission_pb2.StartOperationResponse: + now = Timestamp(seconds=int(datetime.now(UTC).timestamp())) + expires = Timestamp(seconds=now.seconds + 3600) + binding = request.session_binding + return operation_admission_pb2.StartOperationResponse( + provider_instance_id=binding.expected_provider_instance_id, + endpoint_generation=binding.expected_endpoint_generation, + session_id=binding.expected_session_id, + manifest_revision=request.invocation.contract.manifest_revision, + disposition=( + operation_admission_pb2.ADMISSION_DISPOSITION_ACCEPTED_NEW + ), + record=operation_admission_pb2.OperationAdmissionRecord( + operation_id=request.invocation.operation_id, + request_fingerprint=validation.compute_request_fingerprint( + request.invocation + ), + admitted_at=now, + record_expires_at=expires, + admitted_endpoint_generation=binding.expected_endpoint_generation, + ), + ) + + +async def _start_admission_server( + handler: Callable[ + [ + operation_admission_pb2.StartOperationRequest, + grpc.aio.ServicerContext, + ], + Awaitable[operation_admission_pb2.StartOperationResponse | bytes], + ], +) -> tuple[grpc.aio.Server, grpc.aio.Channel]: + method = grpc.unary_unary_rpc_method_handler( + handler, + request_deserializer=( + operation_admission_pb2.StartOperationRequest.FromString + ), + response_serializer=lambda value: ( + value if isinstance(value, bytes) else value.SerializeToString() + ), + ) + server = grpc.aio.server() + server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + "stateforge.sfp.operation_admission.v1." + "OperationAdmissionService", + {"StartOperation": method}, + ), + ) + ) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + channel = grpc.aio.insecure_channel(f"127.0.0.1:{port}") + await channel.channel_ready() + return server, channel + + +async def _coordinator( + path: Path, + client: OperationAdmissionRequester, + *, + attempts: int = 2, +) -> tuple[ + SqliteJournal, + OperationAdmissionCoordinator, + OperationAdmissionStore, +]: + journal = SqliteJournal(path) + await journal.open() + store = journal.operation_admissions(maximum_unresolved_attempts=10) + await store.initialize() + return ( + journal, + OperationAdmissionCoordinator( + store, + attempts=attempts, + retry_initial_delay_seconds=0, + client=client, + ), + store, + ) + + +@pytest.mark.asyncio +async def test_admits_once_and_reuses_one_identity_after_lost_response( + tmp_path: Path, +) -> None: + client = _ScriptedClient( + [ + SfpOperationAdmissionUnavailableError("response lost"), + OperationAdmissionDisposition.RESOLVED_EXISTING, + ] + ) + journal, coordinator, _ = await _coordinator(tmp_path / "agent.db", client) + lease = _lease() + try: + result = await coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=_current, + ) + finally: + await journal.close() + + assert result.disposition is ( + OperationAdmissionDisposition.RESOLVED_EXISTING + ) + assert len(client.requests) == _LOST_RESPONSE_CALL_COUNT + assert ( + client.requests[0].invocation.operation_id + == client.requests[1].invocation.operation_id + ) + assert validation.compute_request_fingerprint( + client.requests[0].invocation + ) == validation.compute_request_fingerprint(client.requests[1].invocation) + + +@pytest.mark.asyncio +async def test_cross_generation_uncertainty_uses_resolve_only( + tmp_path: Path, +) -> None: + client = _ScriptedClient([OperationAdmissionDisposition.RESOLVED_EXISTING]) + journal, coordinator, store = await _coordinator( + tmp_path / "agent.db", client + ) + first_lease = _lease(generation=1) + replacement_lease = _lease(generation=2) + operation_id = await coordinator.prepare( + first_lease, + _request(), + _assessment(first_lease), + assert_current=_current, + ) + attempt = await coordinator.attempt(_PROVIDER_ID, operation_id) + assert attempt is not None + await store.record_transport_attempt( + _PROVIDER_ID, + operation_id, + endpoint_generation=1, + session_id="session-1", + ) + try: + result = await coordinator.submit( + replacement_lease, + operation_id, + _assessment(replacement_lease), + assert_current=_current, + ) + finally: + await journal.close() + + assert result.operation_id == operation_id + assert client.requests[0].admission_mode == ( + operation_admission_pb2.ADMISSION_MODE_RESOLVE_ONLY + ) + + +@pytest.mark.asyncio +async def test_concurrent_submissions_converge_on_one_transport_attempt( + tmp_path: Path, +) -> None: + client = _ScriptedClient([OperationAdmissionDisposition.ACCEPTED_NEW]) + journal, coordinator, _ = await _coordinator(tmp_path / "agent.db", client) + lease = _lease() + assessment = _assessment(lease) + operation_id = await coordinator.prepare( + lease, + _request(), + assessment, + assert_current=_current, + ) + try: + first, second = await asyncio.gather( + coordinator.submit( + lease, + operation_id, + assessment, + assert_current=_current, + ), + coordinator.submit( + lease, + operation_id, + assessment, + assert_current=_current, + ), + ) + finally: + await journal.close() + + assert first == second + assert first.operation_id == operation_id + assert len(client.requests) == 1 + + +@pytest.mark.asyncio +async def test_storage_failure_is_sanitized_before_provider_contact() -> None: + sensitive_value = "sensitive-operation-value" + + class _FailingStore: + async def create_intent(self, *args, **kwargs): + del args, kwargs + raise JournalError( + f"database failed while writing {sensitive_value}" + ) + + client = _ScriptedClient([]) + coordinator = OperationAdmissionCoordinator( + cast(OperationAdmissionStore, _FailingStore()), + client=client, + ) + lease = _lease() + request = replace(_request(), payload={"unit": sensitive_value}) + + with pytest.raises(OperationAdmissionCapacityError) as captured: + await coordinator.admit( + lease, + request, + _assessment(lease), + assert_current=_current, + ) + + assert sensitive_value not in str(captured.value) + assert sensitive_value not in repr(captured.value) + assert client.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "admission_request", + [ + _request(unit=""), + _request(roles=("stateforge.test.profile.v1.undeclared",)), + ], +) +async def test_preflight_rejects_invalid_payload_or_profile( + tmp_path: Path, + admission_request: OperationAdmissionRequest, +) -> None: + client = _ScriptedClient([]) + journal, coordinator, _ = await _coordinator(tmp_path / "agent.db", client) + lease = _lease() + if admission_request.payload["unit"] == "": + lease = replace( + lease, + operation_declarations={ + _OPERATION: ManifestOperationDeclaration( + input_schema={ + **_INPUT_SCHEMA, + "properties": { + "unit": {"type": "string", "minLength": 1} + }, + }, + implements=frozenset({_PROFILE}), + ) + }, + ) + try: + with pytest.raises(OperationAdmissionPreconditionError): + await coordinator.admit( + lease, + admission_request, + _assessment(lease), + assert_current=_current, + ) + finally: + await journal.close() + + assert client.requests == [] + + +@pytest.mark.asyncio +async def test_cancellation_leaves_durable_unresolved_identity( + tmp_path: Path, +) -> None: + started = asyncio.Event() + release = asyncio.Event() + + class _BlockingClient(OperationAdmissionRequester): + async def start( + self, + channel: grpc.aio.Channel, + request: operation_admission_pb2.StartOperationRequest, + ) -> OperationAdmissionResult: + del channel, request + started.set() + await release.wait() + raise AssertionError("cancelled call resumed") + + journal, coordinator, store = await _coordinator( + tmp_path / "agent.db", _BlockingClient() + ) + lease = _lease() + task = asyncio.create_task( + coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=_current, + ) + ) + await started.wait() + task.cancel() + try: + with pytest.raises(asyncio.CancelledError): + await task + unresolved = await store.unresolved() + finally: + release.set() + await journal.close() + + assert len(unresolved) == 1 + assert unresolved[0].attempt_count == 1 + + +@pytest.mark.asyncio +async def test_transport_exhaustion_reports_same_unresolved_identity( + tmp_path: Path, +) -> None: + client = _ScriptedClient( + [SfpOperationAdmissionUnavailableError("unavailable")] + ) + journal, coordinator, _ = await _coordinator( + tmp_path / "agent.db", client, attempts=1 + ) + lease = _lease() + try: + with pytest.raises(OperationAdmissionUnresolvedError) as captured: + await coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=_current, + ) + persisted = await coordinator.attempt( + _PROVIDER_ID, + captured.value.operation_id, + ) + finally: + await journal.close() + + assert persisted is not None + assert persisted.operation_id == captured.value.operation_id + assert persisted.attempt_count == 1 + + +@pytest.mark.asyncio +async def test_wire_client_normalizes_a_canonical_success() -> None: + async def handler(request, context): + del context + return _wire_response(request) + + request = _wire_request() + server, channel = await _start_admission_server(handler) + try: + result = await OperationAdmissionClient().start(channel, request) + finally: + await channel.close() + await server.stop(grace=None) + + assert result.operation_id == UUID(request.invocation.operation_id) + assert result.disposition is OperationAdmissionDisposition.ACCEPTED_NEW + assert result.request_fingerprint == ( + validation.compute_request_fingerprint(request.invocation) + ) + + +@pytest.mark.asyncio +async def test_wire_client_validates_structured_rejection() -> None: + request = _wire_request() + + async def handler(request_message, context): + detail = operation_admission_pb2.OperationAdmissionErrorDetail( + operation_id=request_message.invocation.operation_id, + reason=( + operation_admission_pb2.ADMISSION_ERROR_REASON_PROVIDER_CAPACITY_EXHAUSTED + ), + retryable=True, + retry_after=duration_pb2.Duration(seconds=1), + explanation="Provider capacity is temporarily exhausted.", + ) + packed = any_pb2.Any() + packed.Pack(detail) + await context.abort_with_status( + rpc_status.to_status( + status_pb2.Status( + code=grpc.StatusCode.RESOURCE_EXHAUSTED.value[0], + message="Operation Admission rejected", + details=[packed], + ) + ) + ) + raise AssertionError("abort returned") + + server, channel = await _start_admission_server(handler) + try: + with pytest.raises(SfpOperationAdmissionRejectedError) as captured: + await OperationAdmissionClient().start(channel, request) + finally: + await channel.close() + await server.stop(grace=None) + + assert captured.value.rejection.operation_id == UUID( + request.invocation.operation_id + ) + assert captured.value.rejection.retryable + assert captured.value.rejection.retry_after == timedelta(seconds=1) + + +@pytest.mark.asyncio +async def test_wire_client_rejects_stale_or_malformed_provider_data() -> None: + async def stale_handler(request, context): + del request + await context.abort( + grpc.StatusCode.FAILED_PRECONDITION, + "stale generation", + ) + raise AssertionError("abort returned") + + stale_server, stale_channel = await _start_admission_server(stale_handler) + try: + with pytest.raises(SfpOperationAdmissionStaleError): + await OperationAdmissionClient().start( + stale_channel, + _wire_request(), + ) + finally: + await stale_channel.close() + await stale_server.stop(grace=None) + + async def malformed_handler(request, context): + del request, context + return b"not-a-protobuf" + + malformed_server, malformed_channel = await _start_admission_server( + malformed_handler + ) + try: + with pytest.raises(SfpOperationAdmissionProtocolError): + await OperationAdmissionClient().start( + malformed_channel, + _wire_request(), + ) + finally: + await malformed_channel.close() + await malformed_server.stop(grace=None) + + +@pytest.mark.asyncio +async def test_wire_client_rejects_oversized_response_before_decoding() -> ( + None +): + assert _decode_response(b"x" * _OVERSIZED_RESPONSE_LENGTH).limit_exceeded + + async def handler(request, context): + del request, context + return b"x" * _OVERSIZED_RESPONSE_LENGTH + + server, channel = await _start_admission_server(handler) + try: + with pytest.raises(SfpOperationAdmissionLimitError): + await OperationAdmissionClient().start(channel, _wire_request()) + finally: + await channel.close() + await server.stop(grace=None) diff --git a/tests/test_operation_admission_store.py b/tests/test_operation_admission_store.py new file mode 100644 index 0000000..6e887d0 --- /dev/null +++ b/tests/test_operation_admission_store.py @@ -0,0 +1,214 @@ +"""Durable Agent-owned Operation Admission intent tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import UUID, uuid4 + +import pytest +from google.protobuf.json_format import ParseDict +from google.protobuf.timestamp_pb2 import Timestamp +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, + validation, +) + +from stateforge_agent._journal import SqliteJournal +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionDisposition, + OperationAdmissionResult, + OperationRejection, +) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, + OperationAttemptState, + OperationIntentDecision, +) + +_PROVIDER_ID = UUID("7f7db7d8-68b4-4f71-9769-065b5c4f941c") +_OTHER_PROVIDER_ID = UUID("1d40d0d5-ff2a-4780-bc3d-c4a7c55fc3a2") +_MANIFEST_REVISION = "sha256:" + ("a" * 64) +_ATTEMPTED_ENDPOINT_GENERATION = 4 + + +def _invocation( + operation_id: UUID | None = None, + *, + unit: str = "nginx.service", +) -> operation_admission_pb2.OperationInvocation: + invocation = operation_admission_pb2.OperationInvocation( + operation_id=str(operation_id or uuid4()), + operation_identifier="stateforge.test/restart@1", + execution_deadline=Timestamp( + seconds=int((datetime.now(UTC) + timedelta(minutes=5)).timestamp()) + ), + contract=operation_admission_pb2.OperationContractContext( + manifest_revision=_MANIFEST_REVISION, + ), + ) + ParseDict({"unit": unit}, invocation.payload) + ParseDict({"requester": "test"}, invocation.execution_context) + return invocation + + +async def _store( + path: Path, + *, + capacity: int = 10, +) -> tuple[SqliteJournal, OperationAdmissionStore]: + journal = SqliteJournal(path) + await journal.open() + store = journal.operation_admissions( + maximum_unresolved_attempts=capacity, + ) + await store.initialize() + return journal, store + + +@pytest.mark.asyncio +async def test_intent_is_immutable_and_survives_journal_restart( + tmp_path: Path, +) -> None: + path = tmp_path / "agent.db" + operation_id = uuid4() + invocation = _invocation(operation_id) + fingerprint = validation.compute_request_fingerprint(invocation) + journal, store = await _store(path) + + created = await store.create_intent( + _PROVIDER_ID, + invocation, + fingerprint, + ) + duplicate = await store.create_intent( + _PROVIDER_ID, + invocation, + fingerprint, + ) + conflicting = _invocation(operation_id, unit="sshd.service") + conflict = await store.create_intent( + _PROVIDER_ID, + conflicting, + validation.compute_request_fingerprint(conflicting), + ) + await journal.close() + + reopened, reopened_store = await _store(path) + try: + persisted = await reopened_store.get(_PROVIDER_ID, operation_id) + finally: + await reopened.close() + + assert created.decision is OperationIntentDecision.CREATED + assert duplicate.decision is OperationIntentDecision.EXISTING + assert conflict.decision is OperationIntentDecision.CONFLICT + assert persisted is not None + assert persisted.request_fingerprint == fingerprint + assert persisted.invocation == invocation + + +@pytest.mark.asyncio +async def test_unresolved_capacity_and_provider_identity_are_isolated( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db", capacity=2) + first = _invocation() + second = _invocation() + try: + accepted = await store.create_intent( + _PROVIDER_ID, + first, + validation.compute_request_fingerprint(first), + ) + same_identity_other_provider = await store.create_intent( + _OTHER_PROVIDER_ID, + first, + validation.compute_request_fingerprint(first), + ) + exhausted = await store.create_intent( + _PROVIDER_ID, + second, + validation.compute_request_fingerprint(second), + ) + finally: + await journal.close() + + assert accepted.decision is OperationIntentDecision.CREATED + assert ( + same_identity_other_provider.decision + is OperationIntentDecision.CREATED + ) + assert exhausted.decision is OperationIntentDecision.CAPACITY_EXHAUSTED + + +@pytest.mark.asyncio +async def test_transport_success_and_rejection_are_durable( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db") + accepted_invocation = _invocation() + rejected_invocation = _invocation() + accepted_fingerprint = validation.compute_request_fingerprint( + accepted_invocation + ) + rejected_fingerprint = validation.compute_request_fingerprint( + rejected_invocation + ) + try: + await store.create_intent( + _PROVIDER_ID, + accepted_invocation, + accepted_fingerprint, + ) + await store.create_intent( + _PROVIDER_ID, + rejected_invocation, + rejected_fingerprint, + ) + attempted = await store.record_transport_attempt( + _PROVIDER_ID, + UUID(accepted_invocation.operation_id), + endpoint_generation=_ATTEMPTED_ENDPOINT_GENERATION, + session_id="session-a", + ) + now = datetime.now(UTC) + accepted = await store.record_accepted( + OperationAdmissionResult( + provider_instance_id=_PROVIDER_ID, + endpoint_generation=_ATTEMPTED_ENDPOINT_GENERATION, + session_id="session-a", + manifest_revision=_MANIFEST_REVISION, + operation_id=UUID(accepted_invocation.operation_id), + request_fingerprint=accepted_fingerprint, + disposition=OperationAdmissionDisposition.ACCEPTED_NEW, + admitted_at=now, + record_expires_at=now + timedelta(hours=1), + admitted_endpoint_generation=_ATTEMPTED_ENDPOINT_GENERATION, + ) + ) + rejected = await store.record_rejected( + _PROVIDER_ID, + UUID(rejected_invocation.operation_id), + OperationRejection( + operation_id=UUID(rejected_invocation.operation_id), + reason="ADMISSION_ERROR_REASON_PROVIDER_UNAVAILABLE", + retryable=True, + retry_after=timedelta(seconds=2), + explanation="Provider is draining.", + grpc_status="UNAVAILABLE", + ), + ) + unresolved = await store.unresolved() + finally: + await journal.close() + + assert attempted.attempt_count == 1 + assert attempted.last_endpoint_generation == _ATTEMPTED_ENDPOINT_GENERATION + assert accepted.state is OperationAttemptState.ACCEPTED + assert accepted.result is not None + assert accepted.result.request_fingerprint == accepted_fingerprint + assert rejected.state is OperationAttemptState.REJECTED + assert rejected.rejection is not None + assert rejected.rejection.retry_after == timedelta(seconds=2) + assert unresolved == () diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index 3f778fd..8254149 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -10,7 +10,7 @@ DEPENDENCY_COUNT = 2 HEALTH_SCENARIO_PROVIDER_COUNT = 3 MANIFEST_ASSERTION_COUNT = 6 -AVAILABILITY_ASSERTION_COUNT = 10 +AVAILABILITY_ASSERTION_COUNT = 11 def read(path: Path) -> str: @@ -435,6 +435,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "agent/assert-availability", "agent/assert-capability-boundary", "agent/assert-capability-assessment", + "agent/assert-operation-admission", "agent/assert-trigger-publication", "agent/collect-evidence", "agent/install", @@ -453,6 +454,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "rpm/test-installed/destructive/uninstall-reinstall", "rpm/test-installed/health/provider-availability", "rpm/test-installed/lifecycle/multiple-providers", + "rpm/test-installed/operations/admission", "rpm/test-installed/setup", "rpm/test-installed/smoke/manifest-exchange", "rpm/test-installed/triggers/publication", @@ -508,6 +510,7 @@ def test_each_installed_scenario_is_an_independent_action() -> None: assert tiers == { "smoke", "lifecycle", + "operations", "health", "adversarial", "capability", @@ -635,6 +638,59 @@ def test_installed_capability_scenario_uses_real_artifacts_and_processes() -> ( assert "manifest_revision" in boundary_assertion +def test_installed_operation_admission_uses_real_idempotent_dispatch() -> None: + """Keep A5 evidence on exact packages and Provider-owned execution.""" + workflow = read(WORKFLOWS / "rpm-installed-test.yml") + scenario = read( + ACTIONS + / "rpm" + / "test-installed" + / "operations" + / "admission" + / "action.yml" + ) + assertion_action = read( + ACTIONS / "agent" / "assert-operation-admission" / "action.yml" + ) + assertion = read( + ACTIONS / "agent" / "assert-operation-admission" / "assert.py" + ) + fixture = read( + ACTIONS / "provider" / "create-fixture" / "fixtures" / "manifest.py" + ) + provider_start = read(ACTIONS / "provider" / "start" / "action.yml") + + assert "operation-admission" in workflow + assert "./.github/actions/rpm/test-installed/setup" in scenario + assert "./.github/actions/provider/start" in scenario + assert "./.github/actions/agent/assert-availability" in scenario + assert "./.github/actions/agent/assert-operation-admission" in scenario + assert 'operation-drop-response-once: "true"' in scenario + assert "provider-identity: stateforge.test.operation_admission" in scenario + assert "STATEFORGE_TEST_OPERATION_DROP_RESPONSE_ONCE" in provider_start + assert "OperationAdmissionService.StartOperation" in fixture + assert "response = await original(service, request, context)" in fixture + assert "grpc.StatusCode.UNAVAILABLE" in fixture + assert "operation-admission-response:dropped" in fixture + assert "operation-admission-response:returned" in fixture + assert "/usr/bin/python3" in assertion_action + assert "ProviderSessionManager" in assertion + assert "OperationAdmissionCoordinator" in assertion + assert "SqliteJournal" in assertion + assert "admit_operation" in assertion + assert "RESOLVED_EXISTING" in assertion + assert '_events(path).count("operation:inspect") == 1' in assertion + assert '"restart",' in assertion + assert "_AGENT_UNIT," in assertion + assert "Agent restart lost the admission record" in assertion + assert "request_identity_preserved" in assertion + assert 'version("stateforge-protocol")' in assertion + assert 'version("stateforge-provider")' in assertion + assert "operation-events.txt" in assertion_action + assert "target-sha256.txt" in assertion_action + assert "grpc.unary_unary_rpc_method_handler" not in assertion + + def test_installed_trigger_scenario_uses_public_api_and_journal() -> None: """Keep A4 evidence on exact packages and the durable Agent boundary.""" workflow = read(WORKFLOWS / "rpm-installed-test.yml") diff --git a/uv.lock b/uv.lock index 76246f5..b5305f2 100644 --- a/uv.lock +++ b/uv.lock @@ -431,6 +431,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.76.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/b6/bc095b23a3e0d978e75c8c18cabe4c11f791837410670e4486f0d539bfba/googleapis_common_protos-1.76.0.tar.gz", hash = "sha256:71db0e44a1d4c2dd024ebad44ca67129cc1e2892e4861df466764749d5ef8642", size = 156250, upload-time = "2026-10-22T23:43:36.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6d/78dc67a9e779e594b49bf62604034d3a6aa428a55afc729cafda4b7333e1/googleapis_common_protos-1.76.0-py3-none-any.whl", hash = "sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac", size = 307707, upload-time = "2026-10-22T23:43:10.445Z" }, +] + [[package]] name = "griffelib" version = "2.4.0" @@ -514,6 +526,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/7c/4eb98f1e644b95704ac97693995a8158bd43c3c4030b63c9e980dab7bacb/grpcio_health_checking-1.84.0-py3-none-any.whl", hash = "sha256:3eab9b688e5e09bd1d584c296d3440358f11817388db87b807edd61f45b464d1", size = 19088, upload-time = "2026-09-14T07:09:00.21Z" }, ] +[[package]] +name = "grpcio-status" +version = "1.84.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/45/f80309cdb6a7dbf8f65e2082dd2ddc9797ba7180516a73c54d966ba632c4/grpcio_status-1.84.0.tar.gz", hash = "sha256:5caf28ba7184b81f618b5f7f094859fd2541bf429d2189bbbcd715c9c2cdcee2", size = 14015, upload-time = "2026-09-14T07:10:29.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/c4/3a77e4273e866b1b0c412afd80882e95941a37170032b5109d847c501124/grpcio_status-1.84.0-py3-none-any.whl", hash = "sha256:0c182ca0d6e60acbfd0e14499cf39a155e4827a1c3fd9f7638e49af15a74c30a", size = 14639, upload-time = "2026-09-14T07:10:15.175Z" }, +] + [[package]] name = "idna" version = "3.20" @@ -1447,6 +1473,7 @@ source = { editable = "." } dependencies = [ { name = "grpcio" }, { name = "grpcio-health-checking" }, + { name = "grpcio-status" }, { name = "jsonschema" }, { name = "stateforge-protocol" }, { name = "watchfiles" }, @@ -1462,6 +1489,7 @@ dev = [ { name = "ruff" }, { name = "types-grpcio" }, { name = "types-grpcio-health-checking" }, + { name = "types-grpcio-status" }, { name = "types-jsonschema" }, { name = "types-protobuf" }, ] @@ -1476,6 +1504,7 @@ docs = [ requires-dist = [ { name = "grpcio", specifier = ">=1.83" }, { name = "grpcio-health-checking", specifier = ">=1.83" }, + { name = "grpcio-status", specifier = ">=1.83" }, { name = "jsonschema", specifier = ">=4.25,<5" }, { name = "stateforge-protocol", git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=e4d67e8fd7abbb077e09c41664e6f2945760037e" }, { name = "watchfiles", specifier = ">=1.2,<2" }, @@ -1491,6 +1520,7 @@ dev = [ { name = "ruff", specifier = ">=0.12" }, { name = "types-grpcio" }, { name = "types-grpcio-health-checking" }, + { name = "types-grpcio-status" }, { name = "types-jsonschema" }, { name = "types-protobuf" }, ] @@ -1536,6 +1566,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/de/eca76e380f7e3b93bd7032847e4e9d43904e604b5782c83be9cf733be962/types_grpcio_health_checking-1.0.0.20261021-py3-none-any.whl", hash = "sha256:bac5319aa52e2012f745dba9c125f77326e60db14d60e15cff268dc8759387f5", size = 9828, upload-time = "2026-10-21T06:42:10.574Z" }, ] +[[package]] +name = "types-grpcio-status" +version = "1.0.0.20260408" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-grpcio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/ff/a0270f7083f76167030dacc1eaa81f9424a450b1828733c7b7f1607e7d40/types_grpcio_status-1.0.0.20260408.tar.gz", hash = "sha256:8c8b5a5fd5e661488bb3dfb924fd3562892bae01b9a24e607a26bb0029f3fb71", size = 7092, upload-time = "2026-04-08T04:35:18.169Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/b1/dcdd89efd40eb4a4c2458681a53faf057096db8e2b52a4f94f2bb935aef0/types_grpcio_status-1.0.0.20260408-py3-none-any.whl", hash = "sha256:5d421512b6a9ae5d0f84cde9c251394da5bd77ddc96e122221ac23c8e1ce3acf", size = 7859, upload-time = "2026-04-08T04:35:17.434Z" }, +] + [[package]] name = "types-jsonschema" version = "4.26.0.20261006" From 38a0a581635d741c74048c26632192f6e084706f Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Sun, 1 Nov 2026 16:27:28 +0200 Subject: [PATCH 2/5] test(sfp): expand operation admission coverage --- .../assert-operation-admission/action.yml | 28 +- .../assert-operation-admission/assert.py | 362 ++++++++++++++---- .../create-fixture/fixtures/manifest.py | 35 +- .../operations/admission/action.yml | 55 ++- docs/ci-cd.md | 12 +- docs/sfp-operation-admission.md | 22 +- tests/test_operation_admission.py | 325 +++++++++++++++- tests/test_operation_admission_store.py | 32 ++ tests/test_rpm_automation.py | 31 +- 9 files changed, 785 insertions(+), 117 deletions(-) diff --git a/.github/actions/agent/assert-operation-admission/action.yml b/.github/actions/agent/assert-operation-admission/action.yml index 2be9428..a7a54f7 100644 --- a/.github/actions/agent/assert-operation-admission/action.yml +++ b/.github/actions/agent/assert-operation-admission/action.yml @@ -9,9 +9,19 @@ inputs: description: Provider installation identity published by the fixture. required: true provider-state-name: - description: Fixture state directory containing operation evidence. + description: Primary fixture state directory containing operation evidence. required: false default: provider-a + provider-unit-name: + description: Primary Provider systemd unit replaced during recovery. + required: true + secondary-provider-instance-id: + description: Second Provider identity used to prove isolation. + required: true + secondary-provider-state-name: + description: Secondary fixture state directory containing evidence. + required: false + default: provider-b evidence-directory: description: Host directory receiving sanitized admission evidence. required: true @@ -27,12 +37,16 @@ runs: EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} PROVIDER_INSTANCE_ID: ${{ inputs.provider-instance-id }} PROVIDER_STATE_NAME: ${{ inputs.provider-state-name }} + PROVIDER_UNIT_NAME: ${{ inputs.provider-unit-name }} + SECONDARY_PROVIDER_INSTANCE_ID: ${{ inputs.secondary-provider-instance-id }} + SECONDARY_PROVIDER_STATE_NAME: ${{ inputs.secondary-provider-state-name }} run: | set -euo pipefail mkdir -p "${EVIDENCE_DIRECTORY}" verifier_directory="/opt/stateforge-agent-tests/operation-admission" verifier="${verifier_directory}/assert.py" state_directory="/var/lib/stateforge-provider-fixtures/${PROVIDER_STATE_NAME}" + secondary_state_directory="/var/lib/stateforge-provider-fixtures/${SECONDARY_PROVIDER_STATE_NAME}" docker exec "${CONTAINER_NAME}" mkdir -p "${verifier_directory}" docker cp "${ACTION_PATH}/assert.py" \ "${CONTAINER_NAME}:${verifier}" @@ -40,6 +54,10 @@ runs: docker exec "${CONTAINER_NAME}" /usr/bin/python3 "${verifier}" \ --provider-instance-id "${PROVIDER_INSTANCE_ID}" \ --state-directory "${state_directory}" \ + --provider-unit-name "${PROVIDER_UNIT_NAME}" \ + --secondary-provider-instance-id \ + "${SECONDARY_PROVIDER_INSTANCE_ID}" \ + --secondary-state-directory "${secondary_state_directory}" \ > "${EVIDENCE_DIRECTORY}/operation-admission.json" status=$? set -e @@ -48,4 +66,12 @@ runs: docker exec "${CONTAINER_NAME}" sha256sum \ "${state_directory}/target-state.txt" \ > "${EVIDENCE_DIRECTORY}/target-sha256.txt" 2>&1 || true + docker exec "${CONTAINER_NAME}" cat \ + "${secondary_state_directory}/events.log" \ + > "${EVIDENCE_DIRECTORY}/secondary-operation-events.txt" \ + 2>&1 || true + docker exec "${CONTAINER_NAME}" sha256sum \ + "${secondary_state_directory}/target-state.txt" \ + > "${EVIDENCE_DIRECTORY}/secondary-target-sha256.txt" \ + 2>&1 || true exit "${status}" diff --git a/.github/actions/agent/assert-operation-admission/assert.py b/.github/actions/agent/assert-operation-admission/assert.py index 3999ab4..ae20186 100644 --- a/.github/actions/agent/assert-operation-admission/assert.py +++ b/.github/actions/agent/assert-operation-admission/assert.py @@ -18,12 +18,16 @@ OperationAdmissionDisposition, OperationAdmissionPreconditionError, OperationAdmissionRequest, + OperationAdmissionUnresolvedError, ProviderAvailabilityState, ProviderSessionManager, SfpCapabilityAssessmentRequestError, SfpVersion, ) -from stateforge_agent._sfp._runtime.descriptor import load_provider_descriptor +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, + load_provider_descriptor, +) from stateforge_agent._sfp._services.operation_admission.coordinator import ( OperationAdmissionCoordinator, ) @@ -33,16 +37,36 @@ _DISCOVERY_DIRECTORY = Path("/run/stateforge-agent/providers") _OPERATION = "stateforge.test/inspect@1" _OPERATION_ATTEMPT_PREFIX = "rpc:operation-admission:" -_EXPECTED_ATTEMPTS = 2 +_EXPECTED_RETRY_ATTEMPTS = 2 +_EXPECTED_PRIMARY_DISPATCHES = 2 +_SENSITIVE_VALUE = "stateforge-a5-secret-must-not-leak" def _arguments() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--provider-instance-id", type=UUID, required=True) parser.add_argument("--state-directory", type=Path, required=True) + parser.add_argument("--provider-unit-name", required=True) + parser.add_argument( + "--secondary-provider-instance-id", + type=UUID, + required=True, + ) + parser.add_argument( + "--secondary-state-directory", + type=Path, + required=True, + ) return parser.parse_args() +def _descriptor(provider_instance_id: UUID) -> ProviderEndpointDescriptor: + return load_provider_descriptor( + _DISCOVERY_DIRECTORY / f"{provider_instance_id}.sfp.json", + discovery_directory=_DISCOVERY_DIRECTORY, + ) + + def _digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -54,25 +78,42 @@ def _events(path: Path) -> tuple[str, ...]: return () -def _wait_for_operation_event(path: Path) -> None: - """Wait outside the event loop for one deterministic fixture dispatch.""" +def _wait_for_operation_count(path: Path, expected: int) -> None: + """Wait outside the event loop for an exact dispatch count.""" deadline = time.monotonic() + 10 while time.monotonic() < deadline: - if _events(path).count("operation:inspect") == 1: + if _events(path).count("operation:inspect") == expected: return time.sleep(0.05) - raise AssertionError("Provider did not dispatch exactly one operation") + raise AssertionError( + f"Provider did not dispatch exactly {expected} operations" + ) -async def _restart_agent() -> None: - """Restart the exact packaged Agent unit under test.""" +async def _restart_unit(unit_name: str) -> None: process = await asyncio.create_subprocess_exec( "/usr/bin/systemctl", "restart", - _AGENT_UNIT, + unit_name, ) if await process.wait() != 0: - raise AssertionError("Installed Agent service restart failed") + raise AssertionError(f"systemd restart failed for {unit_name}") + + +async def _wait_for_replacement( + provider_instance_id: UUID, + previous_generation: int, +) -> ProviderEndpointDescriptor: + async with asyncio.timeout(15): + while True: + try: + descriptor = _descriptor(provider_instance_id) + except (FileNotFoundError, ValueError): + await asyncio.sleep(0.05) + continue + if descriptor.endpoint_generation > previous_generation: + return descriptor + await asyncio.sleep(0.05) async def _wait_until_available( @@ -92,12 +133,14 @@ async def _wait_until_available( async def _manager( journal: SqliteJournal, + *, + attempts: int, ) -> tuple[ProviderSessionManager, OperationAdmissionCoordinator]: store = journal.operation_admissions(maximum_unresolved_attempts=32) await store.initialize() coordinator = OperationAdmissionCoordinator( store, - attempts=_EXPECTED_ATTEMPTS, + attempts=attempts, retry_initial_delay_seconds=0.05, ) manager = ProviderSessionManager( @@ -110,6 +153,21 @@ async def _manager( return manager, coordinator +def _request( + provider_instance_id: UUID, + value: str, + *, + scenario: str, +) -> OperationAdmissionRequest: + return OperationAdmissionRequest( + provider_instance_id=provider_instance_id, + operation_identifier=_OPERATION, + payload={"value": value}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=5), + execution_context={"scenario": scenario}, + ) + + async def _require_local_rejections( manager: ProviderSessionManager, provider_instance_id: UUID, @@ -164,75 +222,204 @@ async def _require_local_rejections( raise AssertionError("Locally rejected operation reached Provider") -async def _run(arguments: argparse.Namespace) -> dict[str, object]: - provider_instance_id: UUID = arguments.provider_instance_id - descriptor = load_provider_descriptor( - _DISCOVERY_DIRECTORY / f"{provider_instance_id}.sfp.json", - discovery_directory=_DISCOVERY_DIRECTORY, - ) - event_log = arguments.state_directory / "events.log" - target = arguments.state_directory / "target-state.txt" - target_before = _digest(target) +async def _run( # noqa: C901, PLR0912, PLR0915 + arguments: argparse.Namespace, +) -> dict[str, object]: + primary_id: UUID = arguments.provider_instance_id + secondary_id: UUID = arguments.secondary_provider_instance_id + primary_descriptor = _descriptor(primary_id) + secondary_descriptor = _descriptor(secondary_id) + primary_events_path = arguments.state_directory / "events.log" + secondary_events_path = arguments.secondary_state_directory / "events.log" + primary_target = arguments.state_directory / "target-state.txt" + secondary_target = arguments.secondary_state_directory / "target-state.txt" + primary_target_before = _digest(primary_target) + secondary_target_before = _digest(secondary_target) journal = SqliteJournal(_AGENT_JOURNAL) await journal.open() - manager, coordinator = await _manager(journal) + manager, coordinator = await _manager( + journal, + attempts=_EXPECTED_RETRY_ATTEMPTS, + ) try: - await manager.establish(descriptor) - await _wait_until_available(manager, provider_instance_id) + await manager.establish(primary_descriptor) + await manager.establish(secondary_descriptor) + await asyncio.gather( + _wait_until_available(manager, primary_id), + _wait_until_available(manager, secondary_id), + ) await _require_local_rejections( manager, - provider_instance_id, - event_log, + primary_id, + primary_events_path, ) - result = await manager.admit_operation( - OperationAdmissionRequest( - provider_instance_id=provider_instance_id, - operation_identifier=_OPERATION, - payload={"value": "admitted-once"}, - execution_deadline=datetime.now(UTC) + timedelta(minutes=5), - execution_context={"scenario": "installed-agent"}, - ) + primary_result, secondary_result = await asyncio.gather( + manager.admit_operation( + _request( + primary_id, + _SENSITIVE_VALUE, + scenario="multi-provider-primary", + ) + ), + manager.admit_operation( + _request( + secondary_id, + "secondary-operation", + scenario="multi-provider-secondary", + ) + ), ) - attempt = await coordinator.attempt( - provider_instance_id, - result.operation_id, + primary_attempt = await coordinator.attempt( + primary_id, + primary_result.operation_id, + ) + secondary_attempt = await coordinator.attempt( + secondary_id, + secondary_result.operation_id, ) finally: await manager.close() - await journal.close() - if result.disposition is not ( + if ( + primary_result.disposition + is not OperationAdmissionDisposition.ACCEPTED_NEW + ): + raise AssertionError("Primary Provider did not admit new work") + if secondary_result.disposition is not ( OperationAdmissionDisposition.RESOLVED_EXISTING ): raise AssertionError("Lost response was not resolved idempotently") - if attempt is None or attempt.result != result: - raise AssertionError("Accepted operation was not durably resolved") - if attempt.attempt_count != _EXPECTED_ATTEMPTS: - raise AssertionError("Admission did not use the expected retry count") - - await asyncio.to_thread(_wait_for_operation_event, event_log) - events = _events(event_log) - request_digests = tuple( + if primary_attempt is None or primary_attempt.result != primary_result: + raise AssertionError("Primary admission was not durably resolved") + if ( + secondary_attempt is None + or secondary_attempt.result != secondary_result + ): + raise AssertionError("Secondary admission was not durably resolved") + if primary_attempt.attempt_count != 1: + raise AssertionError("Primary Provider received a duplicate attempt") + if secondary_attempt.attempt_count != _EXPECTED_RETRY_ATTEMPTS: + raise AssertionError("Secondary retry count is incorrect") + if primary_result.operation_id == secondary_result.operation_id: + raise AssertionError("Provider journals shared an operation identity") + + await asyncio.gather( + asyncio.to_thread(_wait_for_operation_count, primary_events_path, 1), + asyncio.to_thread(_wait_for_operation_count, secondary_events_path, 1), + ) + secondary_events = _events(secondary_events_path) + secondary_digests = tuple( event.removeprefix(_OPERATION_ATTEMPT_PREFIX) - for event in events + for event in secondary_events if event.startswith(_OPERATION_ATTEMPT_PREFIX) ) if ( - len(request_digests) != _EXPECTED_ATTEMPTS - or len(set(request_digests)) != 1 + len(secondary_digests) != _EXPECTED_RETRY_ATTEMPTS + or len(set(secondary_digests)) != 1 ): raise AssertionError("Retry changed the immutable wire request") - expected_responses = { - "operation-admission-response:dropped:accepted_new", - "operation-admission-response:returned:resolved_existing", - } - if not expected_responses.issubset(events): - raise AssertionError("Provider did not expose both admission outcomes") - if _digest(target) != target_before: - raise AssertionError("Generic fixture operation mutated its target") - await _restart_agent() + drop_marker = arguments.state_directory / "drop-next-operation-response" + drop_marker.touch() + replacement_manager, replacement_coordinator = await _manager( + journal, + attempts=1, + ) + try: + await replacement_manager.establish(primary_descriptor) + await _wait_until_available(replacement_manager, primary_id) + try: + await replacement_manager.admit_operation( + _request( + primary_id, + "generation-replacement", + scenario="generation-replacement", + ) + ) + except OperationAdmissionUnresolvedError as error: + unresolved_operation_id = error.operation_id + else: + raise AssertionError("Injected response loss was not unresolved") + unresolved_attempt = await replacement_coordinator.attempt( + primary_id, + unresolved_operation_id, + ) + finally: + await replacement_manager.close() + + if unresolved_attempt is None or unresolved_attempt.attempt_count != 1: + raise AssertionError("Uncertain operation identity was not durable") + await asyncio.to_thread(_wait_for_operation_count, primary_events_path, 2) + + await _restart_unit(arguments.provider_unit_name) + replacement_descriptor = await _wait_for_replacement( + primary_id, + primary_descriptor.endpoint_generation, + ) + recovery_manager, recovery_coordinator = await _manager( + journal, + attempts=1, + ) + try: + await recovery_manager.establish(replacement_descriptor) + await _wait_until_available(recovery_manager, primary_id) + try: + await recovery_manager.resolve_operation( + primary_id, + unresolved_operation_id, + ) + except OperationAdmissionUnresolvedError as error: + if error.operation_id != unresolved_operation_id: + raise AssertionError( + "Replacement changed the unresolved operation identity" + ) from error + else: + raise AssertionError( + "Replacement generation claimed a missing retained record" + ) + recovered_attempt = await recovery_coordinator.attempt( + primary_id, + unresolved_operation_id, + ) + finally: + await recovery_manager.close() + await journal.close() + + if recovered_attempt is None or recovered_attempt.result is not None: + raise AssertionError("Unprovable replacement outcome was not retained") + if recovered_attempt.operation_id != unresolved_operation_id: + raise AssertionError("Replacement changed the operation identity") + if recovered_attempt.last_endpoint_generation != ( + replacement_descriptor.endpoint_generation + ): + raise AssertionError("Resolution attempt used the wrong generation") + + primary_events = _events(primary_events_path) + if ( + primary_events.count("operation:inspect") + != _EXPECTED_PRIMARY_DISPATCHES + ): + raise AssertionError("Replacement generation redispatched prior work") + if "operation-admission-mode:resolve_only" not in primary_events: + raise AssertionError("Replacement recovery did not use RESOLVE_ONLY") + if ( + "operation-admission-response:dropped:accepted_new" + not in primary_events + ): + raise AssertionError( + "Generation test did not lose an accepted response" + ) + if _SENSITIVE_VALUE in "\n".join(primary_events + secondary_events): + raise AssertionError( + "Sensitive operation content reached fixture logs" + ) + if _digest(primary_target) != primary_target_before: + raise AssertionError("Primary fixture operation mutated its target") + if _digest(secondary_target) != secondary_target_before: + raise AssertionError("Secondary fixture operation mutated its target") + + await _restart_unit(_AGENT_UNIT) reopened = SqliteJournal(_AGENT_JOURNAL) await reopened.open() reopened_store = reopened.operation_admissions( @@ -240,30 +427,55 @@ async def _run(arguments: argparse.Namespace) -> dict[str, object]: ) await reopened_store.initialize() try: - persisted = await reopened_store.get( - provider_instance_id, - result.operation_id, + persisted_primary = await reopened_store.get( + primary_id, + primary_result.operation_id, + ) + persisted_secondary = await reopened_store.get( + secondary_id, + secondary_result.operation_id, + ) + persisted_recovered = await reopened_store.get( + primary_id, + unresolved_operation_id, ) finally: await reopened.close() - if persisted is None or persisted.result != result: - raise AssertionError("Agent restart lost the admission record") + if persisted_primary is None or persisted_primary.result != primary_result: + raise AssertionError("Agent restart lost the primary admission record") + if ( + persisted_secondary is None + or persisted_secondary.result != secondary_result + ): + raise AssertionError( + "Agent restart lost the secondary admission record" + ) + if persisted_recovered is None or persisted_recovered.result is not None: + raise AssertionError( + "Agent restart lost the unresolved admission record" + ) return { - "agent_restart_preserved_record": True, + "agent_restart_preserved_records": True, "agent_version": version("stateforge-agent"), - "attempt_count": attempt.attempt_count, - "disposition": result.disposition.value, - "endpoint_generation": result.endpoint_generation, - "fingerprint": result.request_fingerprint, - "manifest_revision": result.manifest_revision, - "operation_id": str(result.operation_id), - "operation_identifier": _OPERATION, - "provider_instance_id": str(result.provider_instance_id), - "provider_version": version("stateforge-provider"), + "generation_replacement": { + "from": primary_descriptor.endpoint_generation, + "operation_id": str(unresolved_operation_id), + "remained_unresolved": True, + "request_identity_preserved": True, + "to": replacement_descriptor.endpoint_generation, + "used_resolve_only": True, + }, + "multi_provider_isolation": { + "primary_provider_instance_id": str(primary_id), + "secondary_provider_instance_id": str(secondary_id), + }, + "primary_disposition": primary_result.disposition.value, "protocol_version": version("stateforge-protocol"), - "request_identity_preserved": True, - "target_sha256": target_before, + "provider_version": version("stateforge-provider"), + "secondary_attempt_count": secondary_attempt.attempt_count, + "secondary_disposition": secondary_result.disposition.value, + "secret_safe_evidence": True, } diff --git a/.github/actions/provider/create-fixture/fixtures/manifest.py b/.github/actions/provider/create-fixture/fixtures/manifest.py index 7c6199f..3584959 100644 --- a/.github/actions/provider/create-fixture/fixtures/manifest.py +++ b/.github/actions/provider/create-fixture/fixtures/manifest.py @@ -313,18 +313,20 @@ async def assess_capabilities( def _install_operation_response_loss_once() -> None: - """Drop the first response only after the real Provider admits work.""" - if ( + """Observe admission and optionally lose a post-dispatch response.""" + original = OperationAdmissionService.StartOperation + drop_response = ( os.environ.get( "STATEFORGE_TEST_OPERATION_DROP_RESPONSE_ONCE", "false", ) - != "true" - ): - return - original = OperationAdmissionService.StartOperation - drop_response = True + == "true" + ) event_log = Path(os.environ["STATEFORGE_TEST_EVENT_LOG"]) + drop_marker = ( + Path(os.environ["STATEFORGE_TEST_STATE_DIRECTORY"]) + / "drop-next-operation-response" + ) async def start_operation( service: OperationAdmissionService, @@ -338,6 +340,16 @@ async def start_operation( digest = hashlib.sha256( request.SerializeToString(deterministic=True) ).hexdigest() + _record_event( + event_log, + f"{_OPERATION_ADMISSION_ATTEMPT_PREFIX}{digest}", + ) + mode = ( + operation_admission_pb2.AdmissionMode.Name(request.admission_mode) + .removeprefix("ADMISSION_MODE_") + .lower() + ) + _record_event(event_log, f"operation-admission-mode:{mode}") response = await original(service, request, context) disposition = ( operation_admission_pb2.AdmissionDisposition.Name( @@ -346,12 +358,11 @@ async def start_operation( .removeprefix("ADMISSION_DISPOSITION_") .lower() ) - _record_event( - event_log, - f"{_OPERATION_ADMISSION_ATTEMPT_PREFIX}{digest}", - ) - if drop_response: + marker_armed = drop_marker.exists() + if drop_response or marker_armed: drop_response = False + if marker_armed: + drop_marker.unlink() _record_event( event_log, f"operation-admission-response:dropped:{disposition}", diff --git a/.github/actions/rpm/test-installed/operations/admission/action.yml b/.github/actions/rpm/test-installed/operations/admission/action.yml index 4109475..e889176 100644 --- a/.github/actions/rpm/test-installed/operations/admission/action.yml +++ b/.github/actions/rpm/test-installed/operations/admission/action.yml @@ -26,32 +26,54 @@ runs: evidence-directory: ${{ inputs.evidence-directory }} fixture-directory: ${{ inputs.fixture-directory }} - - name: Start operation-capable Provider - id: provider + - name: Start primary operation-capable Provider + id: primary uses: ./.github/actions/provider/start with: container-name: ${{ steps.setup.outputs.container-name }} fixture-path: ${{ steps.setup.outputs.fixture-path }} wheel-directory: ${{ steps.setup.outputs.wheel-directory }} provider-name: provider-a - provider-identity: stateforge.test.operation_admission + provider-identity: stateforge.test.operation_admission_primary + + - name: Start secondary operation-capable Provider + id: secondary + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: provider-b + provider-identity: stateforge.test.operation_admission_secondary operation-drop-response-once: "true" - - name: Require Agent readiness + - name: Require primary Agent readiness uses: ./.github/actions/agent/assert-availability with: container-name: ${{ steps.setup.outputs.container-name }} - provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} - endpoint-generation: ${{ steps.provider.outputs.endpoint-generation }} + provider-instance-id: ${{ steps.primary.outputs.provider-instance-id }} + endpoint-generation: ${{ steps.primary.outputs.endpoint-generation }} state: available - journal-cursor: ${{ steps.provider.outputs.agent-journal-cursor }} - evidence-directory: ${{ inputs.evidence-directory }}/availability + journal-cursor: ${{ steps.primary.outputs.agent-journal-cursor }} + evidence-directory: ${{ inputs.evidence-directory }}/primary-availability + + - name: Require secondary Agent readiness + uses: ./.github/actions/agent/assert-availability + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.secondary.outputs.provider-instance-id }} + endpoint-generation: ${{ steps.secondary.outputs.endpoint-generation }} + state: available + journal-cursor: ${{ steps.secondary.outputs.agent-journal-cursor }} + evidence-directory: ${{ inputs.evidence-directory }}/secondary-availability - name: Exercise Operation Admission through the installed Agent uses: ./.github/actions/agent/assert-operation-admission with: container-name: ${{ steps.setup.outputs.container-name }} - provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + provider-instance-id: ${{ steps.primary.outputs.provider-instance-id }} + provider-unit-name: ${{ steps.primary.outputs.unit-name }} + secondary-provider-instance-id: ${{ steps.secondary.outputs.provider-instance-id }} evidence-directory: ${{ inputs.evidence-directory }} - name: Preserve evidence and clean up @@ -60,3 +82,18 @@ runs: with: container-name: ${{ steps.setup.outputs.container-name }} evidence-directory: ${{ inputs.evidence-directory }} + + - name: Reject sensitive operation evidence + if: ${{ always() }} + shell: bash + env: + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + if grep --recursive --fixed-strings --line-number \ + "stateforge-a5-secret-must-not-leak" \ + "${EVIDENCE_DIRECTORY}"; then + printf 'Sensitive Operation Admission content reached evidence.\n' \ + >&2 + exit 1 + fi diff --git a/docs/ci-cd.md b/docs/ci-cd.md index cab588c..497d75e 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -140,10 +140,12 @@ The installed-system workflow currently runs twelve isolated scenarios: - triggers: a real Provider wheel publishes through both the decorated and callback author APIs; the installed Agent validates and journals the events, survives an Agent service restart, and returns a terminal acknowledgement; -- operations: the installed Agent durably creates one immutable admission - intent against a real Provider Python service, survives a deliberately lost - successful response, retries the same identity, proves exactly-once Provider - dispatch, resolves the existing record, and retains it across Agent restart; +- operations: two real Provider Python services prove simultaneous isolation, + durable immutable intents, deliberately lost successful responses, + same-identity retry and exactly-once dispatch. The scenario also replaces a + Provider generation, performs resolve-only recovery without redispatch, + retains unprovable work as unresolved across Agent restart, and rejects any + sensitive payload value found in journals or retained evidence; - adversarial: malformed discovery data is rejected without stopping the Agent; - destructive: the exact RPM is removed, installed again in a clean environment, and completes manifest exchange after reinstallation. @@ -169,7 +171,7 @@ digest of each generated wheel. | `agent/assert-capability-assessment` | Use the RPM-installed Agent package to collect and verify normalized evidence from a separate real Provider process. | | `agent/assert-capability-boundary` | Exercise timeout, malformed response, stale response-fence, and byte-limit behavior through the RPM-installed client. | | `agent/assert-trigger-publication` | Validate canonical, durable Provider publication evidence in the installed Agent journal. | -| `agent/assert-operation-admission` | Verify local preflight, durable intent, retry identity, real Provider idempotency, exactly-once dispatch, and restart persistence. | +| `agent/assert-operation-admission` | Verify local preflight, multi-Provider isolation, durable intent, retry identity, real Provider idempotency, generation-replacement recovery without redispatch, restart persistence, and secret-safe evidence. | | `agent/collect-evidence` | Capture bounded evidence, uninstall the RPM, and clean up. | | `provider/create-fixture` | Invoke the selected Protocol and Provider checkouts' canonical package-build actions and stage their wheels with a selected fixture. | | `provider/start` | Run the fixture as a separate non-root systemd service. | diff --git a/docs/sfp-operation-admission.md b/docs/sfp-operation-admission.md index 49d73ee..4daa764 100644 --- a/docs/sfp-operation-admission.md +++ b/docs/sfp-operation-admission.md @@ -92,16 +92,22 @@ are not copied into logs, exception text, or CI evidence. ## Installed verification The installed-system scenario builds the Agent RPM and exact Protocol and -Provider wheels, then starts the packaged Agent and a real Provider Python -process as separate systemd services. The Provider commits and dispatches one -operation, deliberately drops the first successful response, and returns the -existing record when the Agent retries. +Provider wheels, then starts the packaged Agent and two real Provider Python +processes as separate systemd services. One Provider completes a normal +admission while the other deliberately loses its first successful response +and resolves the Agent's identical retry from its existing record. The scenario proves that: -- malformed or undeclared requests are rejected locally; +- malformed, undeclared, oversized, expired, or unsupported requests are + rejected before unsafe dispatch; +- simultaneous admissions remain isolated to their selected Provider; - one immutable request is sent twice with the same operation UUID and digest; -- the real Provider handler runs exactly once; +- each real Provider handler runs exactly once per admitted identity; - the Agent receives `RESOLVED_EXISTING` after the simulated response loss; -- the accepted record survives an Agent service restart; and -- evidence contains identifiers and hashes, never operation payloads. +- a lost response followed by Provider generation replacement uses + `RESOLVE_ONLY`, preserves the original identity, and remains unresolved when + the replacement process cannot prove retained state; +- accepted and unresolved records survive an Agent service restart; and +- journals and retained CI evidence contain identifiers and hashes, never + sensitive operation payload values. diff --git a/tests/test_operation_admission.py b/tests/test_operation_admission.py index 7842ea3..c9edca9 100644 --- a/tests/test_operation_admission.py +++ b/tests/test_operation_admission.py @@ -29,6 +29,8 @@ CapabilityAssessment, CapabilityAssessmentScope, CapabilityFinding, + CapabilityRequirement, + CapabilityRequirementState, CapabilitySupportState, ) from stateforge_agent._sfp._services.manifest.declaration import ( @@ -37,12 +39,14 @@ from stateforge_agent._sfp._services.operation_admission.client import ( OperationAdmissionClient, OperationAdmissionRequester, + SfpOperationAdmissionError, SfpOperationAdmissionLimitError, SfpOperationAdmissionProtocolError, SfpOperationAdmissionRejectedError, SfpOperationAdmissionStaleError, SfpOperationAdmissionUnavailableError, _decode_response, + validate_request, ) from stateforge_agent._sfp._services.operation_admission.coordinator import ( OperationAdmissionCapacityError, @@ -55,9 +59,11 @@ OperationAdmissionDisposition, OperationAdmissionRequest, OperationAdmissionResult, + OperationRejection, ) from stateforge_agent._sfp._services.operation_admission.store import ( OperationAdmissionStore, + OperationAttemptState, ) _PROVIDER_ID = UUID("7f7db7d8-68b4-4f71-9769-065b5c4f941c") @@ -81,8 +87,7 @@ class _ScriptedClient(OperationAdmissionRequester): def __init__( self, steps: list[ - OperationAdmissionDisposition - | SfpOperationAdmissionUnavailableError + OperationAdmissionDisposition | SfpOperationAdmissionError ], ) -> None: self.steps = steps @@ -566,6 +571,273 @@ async def test_transport_exhaustion_reports_same_unresolved_identity( assert persisted.attempt_count == 1 +@pytest.mark.asyncio +async def test_retryable_rejection_reuses_identity_then_succeeds( + tmp_path: Path, +) -> None: + rejection = OperationRejection( + operation_id=None, + reason="ADMISSION_ERROR_REASON_PROVIDER_CAPACITY_EXHAUSTED", + retryable=True, + retry_after=None, + explanation="Provider capacity is temporarily exhausted.", + grpc_status=grpc.StatusCode.RESOURCE_EXHAUSTED.name, + ) + client = _ScriptedClient( + [ + SfpOperationAdmissionRejectedError(rejection), + OperationAdmissionDisposition.ACCEPTED_NEW, + ] + ) + journal, coordinator, _ = await _coordinator(tmp_path / "agent.db", client) + lease = _lease() + try: + result = await coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=_current, + ) + finally: + await journal.close() + + assert result.disposition is OperationAdmissionDisposition.ACCEPTED_NEW + assert len(client.requests) == _LOST_RESPONSE_CALL_COUNT + assert ( + client.requests[0].invocation.operation_id + == client.requests[1].invocation.operation_id + ) + + +@pytest.mark.asyncio +async def test_non_retryable_rejection_is_durable_and_not_retried( + tmp_path: Path, +) -> None: + rejection = OperationRejection( + operation_id=None, + reason="ADMISSION_ERROR_REASON_PROVIDER_UNAVAILABLE", + retryable=False, + retry_after=None, + explanation="Provider is draining.", + grpc_status=grpc.StatusCode.FAILED_PRECONDITION.name, + ) + client = _ScriptedClient([SfpOperationAdmissionRejectedError(rejection)]) + journal, coordinator, store = await _coordinator( + tmp_path / "agent.db", client + ) + lease = _lease() + try: + with pytest.raises(SfpOperationAdmissionRejectedError) as captured: + await coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=_current, + ) + assert captured.value.rejection.operation_id is None + operation_id = UUID(client.requests[0].invocation.operation_id) + attempts = await store.unresolved() + assert attempts == () + rejected = await store.get(_PROVIDER_ID, operation_id) + with pytest.raises(SfpOperationAdmissionRejectedError): + await coordinator.submit( + lease, + operation_id, + _assessment(lease), + assert_current=_current, + ) + finally: + await journal.close() + + assert rejected is not None + assert rejected.state is OperationAttemptState.REJECTED + assert rejected.rejection is not None + assert rejected.rejection.reason == rejection.reason + assert not rejected.rejection.retryable + assert len(client.requests) == 1 + + +@pytest.mark.asyncio +async def test_expired_execution_deadline_never_contacts_provider( + tmp_path: Path, +) -> None: + client = _ScriptedClient([]) + journal, coordinator, store = await _coordinator( + tmp_path / "agent.db", client + ) + lease = _lease() + request = replace( + _request(), + execution_deadline=datetime.now(UTC) - timedelta(seconds=1), + ) + try: + with pytest.raises(SfpOperationAdmissionRejectedError): + await coordinator.admit( + lease, + request, + _assessment(lease), + assert_current=_current, + ) + unresolved = await store.unresolved() + finally: + await journal.close() + + assert unresolved == () + assert client.requests == [] + + +@pytest.mark.asyncio +async def test_post_accept_storage_failure_leaves_identity_unresolved( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = _ScriptedClient([OperationAdmissionDisposition.ACCEPTED_NEW]) + journal, coordinator, store = await _coordinator( + tmp_path / "agent.db", client + ) + + async def fail_to_record(result: OperationAdmissionResult): + del result + raise JournalError("accepted result could not be stored") + + monkeypatch.setattr(store, "record_accepted", fail_to_record) + lease = _lease() + try: + with pytest.raises(OperationAdmissionUnresolvedError) as captured: + await coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=_current, + ) + attempt = await store.get(_PROVIDER_ID, captured.value.operation_id) + finally: + await journal.close() + + assert attempt is not None + assert attempt.state is OperationAttemptState.UNRESOLVED + assert attempt.result is None + assert len(client.requests) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stale_after_call", + [2, 4], + ids=["after-intent-before-transport", "after-response-before-commit"], +) +async def test_currentness_changes_cannot_become_authoritative( + tmp_path: Path, + stale_after_call: int, +) -> None: + client = _ScriptedClient([OperationAdmissionDisposition.ACCEPTED_NEW]) + journal, coordinator, store = await _coordinator( + tmp_path / f"agent-{stale_after_call}.db", client + ) + calls = 0 + + async def current(lease: OperationAdmissionLease) -> None: + nonlocal calls + del lease + calls += 1 + if calls == stale_after_call: + raise SfpOperationAdmissionStaleError("session changed") + + lease = _lease() + try: + with pytest.raises(SfpOperationAdmissionStaleError): + await coordinator.admit( + lease, + _request(), + _assessment(lease), + assert_current=current, + ) + unresolved = await store.unresolved() + finally: + await journal.close() + + assert len(unresolved) == 1 + assert unresolved[0].result is None + assert len(client.requests) == ( + 0 if stale_after_call == _LOST_RESPONSE_CALL_COUNT else 1 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "evidence_case", + [ + "unsupported", + "indeterminate", + "unsatisfied-requirement", + "stale-scope", + "wrong-operation", + "wrong-context", + ], +) +async def test_assessment_must_exactly_support_admission_scope( + tmp_path: Path, + evidence_case: str, +) -> None: + client = _ScriptedClient([]) + journal, coordinator, store = await _coordinator( + tmp_path / f"agent-{evidence_case}.db", client + ) + lease = _lease() + assessment = _assessment(lease) + finding = assessment.findings[0] + if evidence_case == "unsupported": + finding = replace( + finding, + support_state=CapabilitySupportState.UNSUPPORTED, + ) + elif evidence_case == "indeterminate": + finding = replace( + finding, + support_state=CapabilitySupportState.INDETERMINATE, + ) + elif evidence_case == "unsatisfied-requirement": + finding = replace( + finding, + support_state=CapabilitySupportState.SUPPORTED_WITH_REQUIREMENTS, + requirements=( + CapabilityRequirement( + identifier="runtime.target.readable", + state=CapabilityRequirementState.UNSATISFIED, + description="Target is not readable.", + version_constraint="", + ), + ), + ) + elif evidence_case == "wrong-operation": + finding = replace( + finding, + operation_identifier="stateforge.test/other@1", + ) + scope = assessment.scope + if evidence_case == "stale-scope": + scope = replace( + scope, endpoint_generation=scope.endpoint_generation + 1 + ) + elif evidence_case == "wrong-context": + scope = replace(scope, execution_context_sha256=b"\x00" * 32) + assessment = replace(assessment, scope=scope, findings=(finding,)) + try: + with pytest.raises(OperationAdmissionPreconditionError): + await coordinator.admit( + lease, + _request(), + assessment, + assert_current=_current, + ) + unresolved = await store.unresolved() + finally: + await journal.close() + + assert unresolved == () + assert client.requests == [] + + @pytest.mark.asyncio async def test_wire_client_normalizes_a_canonical_success() -> None: async def handler(request, context): @@ -685,3 +957,52 @@ async def handler(request, context): finally: await channel.close() await server.stop(grace=None) + + +def test_wire_client_rejects_oversized_request_before_transport() -> None: + request = _wire_request() + request.invocation.payload["oversized"] = "x" * contract.MAX_REQUEST_BYTES + + with pytest.raises(SfpOperationAdmissionLimitError): + validate_request(request) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mismatch", + [ + "provider-instance", + "endpoint-generation", + "session", + "manifest-revision", + "operation-id", + "fingerprint", + ], +) +async def test_wire_client_rejects_mismatched_success_correlation( + mismatch: str, +) -> None: + async def handler(request, context): + del context + response = _wire_response(request) + if mismatch == "provider-instance": + response.provider_instance_id = str(uuid4()) + elif mismatch == "endpoint-generation": + response.endpoint_generation += 1 + elif mismatch == "session": + response.session_id = "foreign-session" + elif mismatch == "manifest-revision": + response.manifest_revision = "sha256:" + ("b" * 64) + elif mismatch == "operation-id": + response.record.operation_id = str(uuid4()) + elif mismatch == "fingerprint": + response.record.request_fingerprint = "sha256:" + ("b" * 64) + return response + + server, channel = await _start_admission_server(handler) + try: + with pytest.raises(SfpOperationAdmissionProtocolError): + await OperationAdmissionClient().start(channel, _wire_request()) + finally: + await channel.close() + await server.stop(grace=None) diff --git a/tests/test_operation_admission_store.py b/tests/test_operation_admission_store.py index 6e887d0..47d4291 100644 --- a/tests/test_operation_admission_store.py +++ b/tests/test_operation_admission_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from datetime import UTC, datetime, timedelta from pathlib import Path from uuid import UUID, uuid4 @@ -142,6 +143,37 @@ async def test_unresolved_capacity_and_provider_identity_are_isolated( assert exhausted.decision is OperationIntentDecision.CAPACITY_EXHAUSTED +@pytest.mark.asyncio +async def test_concurrent_intent_creation_cannot_exceed_capacity( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db", capacity=1) + first = _invocation() + second = _invocation() + try: + results = await asyncio.gather( + store.create_intent( + _PROVIDER_ID, + first, + validation.compute_request_fingerprint(first), + ), + store.create_intent( + _PROVIDER_ID, + second, + validation.compute_request_fingerprint(second), + ), + ) + unresolved = await store.unresolved() + finally: + await journal.close() + + assert {result.decision for result in results} == { + OperationIntentDecision.CREATED, + OperationIntentDecision.CAPACITY_EXHAUSTED, + } + assert len(unresolved) == 1 + + @pytest.mark.asyncio async def test_transport_success_and_rejection_are_durable( tmp_path: Path, diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index 8254149..20daa93 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -10,7 +10,8 @@ DEPENDENCY_COUNT = 2 HEALTH_SCENARIO_PROVIDER_COUNT = 3 MANIFEST_ASSERTION_COUNT = 6 -AVAILABILITY_ASSERTION_COUNT = 11 +AVAILABILITY_ASSERTION_COUNT = 12 +OPERATION_PROVIDER_COUNT = 2 def read(path: Path) -> str: @@ -665,8 +666,22 @@ def test_installed_operation_admission_uses_real_idempotent_dispatch() -> None: assert "./.github/actions/provider/start" in scenario assert "./.github/actions/agent/assert-availability" in scenario assert "./.github/actions/agent/assert-operation-admission" in scenario + assert ( + scenario.count("./.github/actions/provider/start") + == OPERATION_PROVIDER_COUNT + ) assert 'operation-drop-response-once: "true"' in scenario - assert "provider-identity: stateforge.test.operation_admission" in scenario + assert ( + "provider-identity: stateforge.test.operation_admission_primary" + in scenario + ) + assert ( + "provider-identity: stateforge.test.operation_admission_secondary" + in scenario + ) + assert "secondary-provider-instance-id" in scenario + assert "provider-unit-name" in scenario + assert "Reject sensitive operation evidence" in scenario assert "STATEFORGE_TEST_OPERATION_DROP_RESPONSE_ONCE" in provider_start assert "OperationAdmissionService.StartOperation" in fixture assert "response = await original(service, request, context)" in fixture @@ -679,10 +694,16 @@ def test_installed_operation_admission_uses_real_idempotent_dispatch() -> None: assert "SqliteJournal" in assertion assert "admit_operation" in assertion assert "RESOLVED_EXISTING" in assertion - assert '_events(path).count("operation:inspect") == 1' in assertion + assert "asyncio.gather" in assertion + assert "multi_provider_isolation" in assertion + assert "drop-next-operation-response" in assertion + assert "resolve_operation" in assertion + assert "operation-admission-mode:resolve_only" in assertion + assert "Replacement generation redispatched prior work" in assertion + assert "secret_safe_evidence" in assertion assert '"restart",' in assertion - assert "_AGENT_UNIT," in assertion - assert "Agent restart lost the admission record" in assertion + assert "_restart_unit(_AGENT_UNIT)" in assertion + assert "Agent restart lost the unresolved admission record" in assertion assert "request_identity_preserved" in assertion assert 'version("stateforge-protocol")' in assertion assert 'version("stateforge-provider")' in assertion From 12bcc9759e7f5c0e3f57af105be69ab00bc2aa9b Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Sun, 1 Nov 2026 16:46:35 +0200 Subject: [PATCH 3/5] refactor(journal): organize SQLite backend package --- src/stateforge_agent/_journal/sqlite/__init__.py | 8 ++++++++ .../_journal/{sqlite.py => sqlite/journal.py} | 4 ++-- .../operation_admissions.py} | 0 .../trigger_publications.py} | 0 tests/test_sqlite_journal.py | 2 +- 5 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 src/stateforge_agent/_journal/sqlite/__init__.py rename src/stateforge_agent/_journal/{sqlite.py => sqlite/journal.py} (97%) rename src/stateforge_agent/_journal/{sqlite_operation_admissions.py => sqlite/operation_admissions.py} (100%) rename src/stateforge_agent/_journal/{sqlite_trigger_publications.py => sqlite/trigger_publications.py} (100%) diff --git a/src/stateforge_agent/_journal/sqlite/__init__.py b/src/stateforge_agent/_journal/sqlite/__init__.py new file mode 100644 index 0000000..613e3f5 --- /dev/null +++ b/src/stateforge_agent/_journal/sqlite/__init__.py @@ -0,0 +1,8 @@ +"""Expose the SQLite journal backend.""" + +from stateforge_agent._journal.sqlite.journal import ( + SqliteJournal, + SqliteJournalError, +) + +__all__ = ["SqliteJournal", "SqliteJournalError"] diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite/journal.py similarity index 97% rename from src/stateforge_agent/_journal/sqlite.py rename to src/stateforge_agent/_journal/sqlite/journal.py index 64f1787..e1f58df 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite/journal.py @@ -10,10 +10,10 @@ from typing import TYPE_CHECKING, Self, TypeVar from stateforge_agent._journal.base import Journal, JournalError -from stateforge_agent._journal.sqlite_operation_admissions import ( +from stateforge_agent._journal.sqlite.operation_admissions import ( SqliteOperationAdmissionStore, ) -from stateforge_agent._journal.sqlite_trigger_publications import ( +from stateforge_agent._journal.sqlite.trigger_publications import ( SqliteTriggerPublicationStore, ) from stateforge_agent._sfp._services.operation_admission.store import ( diff --git a/src/stateforge_agent/_journal/sqlite_operation_admissions.py b/src/stateforge_agent/_journal/sqlite/operation_admissions.py similarity index 100% rename from src/stateforge_agent/_journal/sqlite_operation_admissions.py rename to src/stateforge_agent/_journal/sqlite/operation_admissions.py diff --git a/src/stateforge_agent/_journal/sqlite_trigger_publications.py b/src/stateforge_agent/_journal/sqlite/trigger_publications.py similarity index 100% rename from src/stateforge_agent/_journal/sqlite_trigger_publications.py rename to src/stateforge_agent/_journal/sqlite/trigger_publications.py diff --git a/tests/test_sqlite_journal.py b/tests/test_sqlite_journal.py index 024468c..5d8fd54 100644 --- a/tests/test_sqlite_journal.py +++ b/tests/test_sqlite_journal.py @@ -9,7 +9,7 @@ import pytest from stateforge_agent._journal import SqliteJournal, SqliteJournalError -from stateforge_agent._journal import sqlite as sqlite_journal +from stateforge_agent._journal.sqlite import journal as sqlite_journal _LARGE_PAYLOAD_BYTES = 1024 * 1024 _SHORT_BUSY_TIMEOUT_MILLISECONDS = 1 From 23ec5af4275b832a0ce5db7e1f9790099862c10e Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Sun, 1 Nov 2026 16:58:56 +0200 Subject: [PATCH 4/5] fix(sfp): honor admission retry deadlines --- .../operation_admission/coordinator.py | 11 ++++--- tests/test_operation_admission.py | 33 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py index 4b7db97..f205da2 100644 --- a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py +++ b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py @@ -367,11 +367,12 @@ async def _handle_rejection( ) if not rejection.retryable or attempt_number + 1 >= self._attempts: raise error - await _retry_delay( + if not await _retry_delay( rejection, fallback=self._retry_initial_delay_seconds * (2**attempt_number), deadline=_deadline(attempt.invocation), - ) + ): + raise error async def _retry_transport_error( self, @@ -571,12 +572,14 @@ async def _retry_delay( *, fallback: float, deadline: datetime, -) -> None: +) -> bool: + """Wait for the prescribed delay when another timely attempt can fit.""" delay = ( rejection.retry_after.total_seconds() if rejection.retry_after is not None else fallback ) if datetime.now(UTC).timestamp() + delay >= deadline.timestamp(): - return + return False await asyncio.sleep(delay) + return True diff --git a/tests/test_operation_admission.py b/tests/test_operation_admission.py index c9edca9..1668379 100644 --- a/tests/test_operation_admission.py +++ b/tests/test_operation_admission.py @@ -609,6 +609,39 @@ async def test_retryable_rejection_reuses_identity_then_succeeds( ) +@pytest.mark.asyncio +async def test_retry_after_beyond_execution_deadline_stops_retry( + tmp_path: Path, +) -> None: + rejection = OperationRejection( + operation_id=None, + reason="ADMISSION_ERROR_REASON_PROVIDER_CAPACITY_EXHAUSTED", + retryable=True, + retry_after=timedelta(minutes=1), + explanation="Provider capacity is temporarily exhausted.", + grpc_status=grpc.StatusCode.RESOURCE_EXHAUSTED.name, + ) + client = _ScriptedClient([SfpOperationAdmissionRejectedError(rejection)]) + journal, coordinator, _ = await _coordinator(tmp_path / "agent.db", client) + lease = _lease() + request = replace( + _request(), + execution_deadline=datetime.now(UTC) + timedelta(seconds=1), + ) + try: + with pytest.raises(SfpOperationAdmissionRejectedError): + await coordinator.admit( + lease, + request, + _assessment(lease), + assert_current=_current, + ) + finally: + await journal.close() + + assert len(client.requests) == 1 + + @pytest.mark.asyncio async def test_non_retryable_rejection_is_durable_and_not_retried( tmp_path: Path, From 56f461da2ba91181cd5b5f1706f6f966009712a0 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Sun, 1 Nov 2026 17:02:53 +0200 Subject: [PATCH 5/5] fix(sfp): validate admission rejection semantics --- .../_services/operation_admission/client.py | 5 +- tests/test_operation_admission.py | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/client.py b/src/stateforge_agent/_sfp/_services/operation_admission/client.py index 046c248..f0e5012 100644 --- a/src/stateforge_agent/_sfp/_services/operation_admission/client.py +++ b/src/stateforge_agent/_sfp/_services/operation_admission/client.py @@ -197,7 +197,10 @@ def _canonical_rejection( if not status.details[0].Unpack(detail): return None try: - validation.validate_admission_error_detail(detail) + validation.validate_admission_rejection( + detail, + grpc_status=error.code().name, + ) except validation.OperationAdmissionValidationError as validation_error: raise SfpOperationAdmissionProtocolError( "Operation Admission rejection failed canonical validation" diff --git a/tests/test_operation_admission.py b/tests/test_operation_admission.py index 1668379..1e1dd85 100644 --- a/tests/test_operation_admission.py +++ b/tests/test_operation_admission.py @@ -934,6 +934,52 @@ async def handler(request_message, context): assert captured.value.rejection.retry_after == timedelta(seconds=1) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("grpc_status", "retryable"), + [ + (grpc.StatusCode.FAILED_PRECONDITION, True), + (grpc.StatusCode.UNAVAILABLE, False), + ], + ids=["status-does-not-match-reason", "retryable-does-not-match-reason"], +) +async def test_wire_client_rejects_inconsistent_rejection_semantics( + grpc_status: grpc.StatusCode, + retryable: bool, +) -> None: + request = _wire_request() + + async def handler(request_message, context): + detail = operation_admission_pb2.OperationAdmissionErrorDetail( + operation_id=request_message.invocation.operation_id, + reason=( + operation_admission_pb2.ADMISSION_ERROR_REASON_PROVIDER_UNAVAILABLE + ), + retryable=retryable, + explanation="Provider is temporarily unavailable.", + ) + packed = any_pb2.Any() + packed.Pack(detail) + await context.abort_with_status( + rpc_status.to_status( + status_pb2.Status( + code=grpc_status.value[0], + message="Operation Admission rejected", + details=[packed], + ) + ) + ) + raise AssertionError("abort returned") + + server, channel = await _start_admission_server(handler) + try: + with pytest.raises(SfpOperationAdmissionProtocolError): + await OperationAdmissionClient().start(channel, request) + finally: + await channel.close() + await server.stop(grace=None) + + @pytest.mark.asyncio async def test_wire_client_rejects_stale_or_malformed_provider_data() -> None: async def stale_handler(request, context):