From d29fdfd91904d47b20b74f03aeebd342772b5ac2 Mon Sep 17 00:00:00 2001 From: Aditya Salunkhe Date: Tue, 15 Sep 2026 01:13:12 +0530 Subject: [PATCH 1/6] Implement Oplog epoch fencing: oplog_metadata, epoch-checked appends, ShardLost relinquish --- Cargo.lock | 1 + docs/src/content/next/deploy.mdx | 4 +- docs/src/content/next/operate/persistence.mdx | 2 +- .../golem/worker/invocation_session.proto | 4 + .../proto/golem/worker/raw_oplog.proto | 1 + golem-common/src/base_model/oplog/mod.rs | 11 +- golem-common/src/cache.rs | 52 ++ golem-common/src/model/oplog/protobuf.rs | 3 + golem-common/src/model/oplog/tests.rs | 70 ++ .../src/oplog/debug_oplog.rs | 30 +- .../src/oplog/debug_oplog_constructor.rs | 6 + .../src/oplog/debug_oplog_service.rs | 3 + .../src/services/debug_service.rs | 52 +- .../src/error/worker_executor.rs | 54 ++ golem-shard-manager/src/sharding/model.rs | 199 +++++ .../src/sharding/shard_management.rs | 12 + golem-shard-manager/tests/shard_management.rs | 55 +- golem-test-framework/Cargo.toml | 3 + golem-test-framework/src/components/mod.rs | 22 + .../src/components/shard_manager/mod.rs | 8 + .../src/components/shard_manager/spawned.rs | 7 + .../src/components/worker_executor/mod.rs | 18 + .../src/components/worker_executor/spawned.rs | 34 + .../components/worker_executor_cluster/mod.rs | 15 + .../worker_executor_cluster/spawned.rs | 8 + golem-test-framework/src/config/benchmark.rs | 1 + golem-test-framework/src/config/env.rs | 32 + golem-test-framework/src/dsl/mod.rs | 7 + golem-worker-executor-test-utils/src/lib.rs | 65 +- golem-worker-executor/benches/oplog_read.rs | 12 +- .../indexed/postgres/002_oplog_metadata.sql | 13 + .../indexed/sqlite/002_oplog_metadata.sql | 7 + .../src/durable_host/concurrent/call.rs | 12 +- .../src/durable_host/concurrent/delivery.rs | 13 +- .../durable_host/concurrent/drop_events.rs | 2 +- .../src/durable_host/concurrent/tests.rs | 123 ++- .../src/durable_host/durable_session.rs | 56 +- .../src/durable_host/durable_stream.rs | 249 +++--- .../durable_host/durable_stream/metadata.rs | 29 +- .../src/durable_host/golem/v1x.rs | 4 +- .../src/durable_host/logging/policy.rs | 18 +- golem-worker-executor/src/durable_host/mod.rs | 77 +- .../src/durable_host/p3/http/request_body.rs | 5 +- .../src/durable_host/p3/http/response_body.rs | 9 +- .../src/durable_host/p3/http/test_support.rs | 24 +- .../src/durable_host/replay_state/tests.rs | 96 ++- .../src/durable_host/suspendable_wait.rs | 32 +- .../src/durable_host/wasm_rpc/mod.rs | 2 +- .../src/grpc/invocation_session.rs | 33 + golem-worker-executor/src/grpc/mod.rs | 49 +- golem-worker-executor/src/lib.rs | 84 ++ golem-worker-executor/src/model/mod.rs | 104 ++- .../src/model/public_oplog/mod.rs | 1 + .../src/model/public_oplog/tests.rs | 124 ++- .../src/model/public_oplog/wit.rs | 4 + .../services/active_agents/memory_probe.rs | 16 +- .../src/services/active_agents/mod.rs | 55 +- .../src/services/oplog/compressed.rs | 2 +- .../src/services/oplog/ephemeral.rs | 38 +- .../src/services/oplog/mod.rs | 188 +++- .../src/services/oplog/multilayer.rs | 34 +- .../src/services/oplog/plugin.rs | 337 +++++--- .../src/services/oplog/primary.rs | 416 +++++++-- .../src/services/oplog/rate_limited.rs | 47 +- .../src/services/oplog/tests.rs | 805 +++++++++++++++--- golem-worker-executor/src/services/rpc.rs | 2 + .../src/services/shard_manager.rs | 33 + golem-worker-executor/src/services/worker.rs | 9 +- .../services/worker/session_index_tests.rs | 138 ++- .../src/services/worker_fork.rs | 19 +- .../src/storage/indexed/memory.rs | 30 + .../src/storage/indexed/mod.rs | 136 ++- .../src/storage/indexed/multi_sqlite.rs | 64 +- .../src/storage/indexed/postgres.rs | 174 +++- .../src/storage/indexed/redis.rs | 3 + .../src/storage/indexed/sqlite.rs | 169 +++- golem-worker-executor/src/worker/instance.rs | 19 +- .../src/worker/invocation.rs | 7 + .../src/worker/invocation_loop.rs | 30 + golem-worker-executor/src/worker/mod.rs | 277 +++++- .../src/worker/state_actor.rs | 76 +- golem-worker-executor/src/worker/status.rs | 5 + .../tests/indexed_storage.rs | 512 ++++++++++- .../src/service/worker/client.rs | 82 +- integration-tests/tests/sharding.rs | 156 +++- local-run/start.sh | 6 +- 86 files changed, 4855 insertions(+), 991 deletions(-) create mode 100644 golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql create mode 100644 golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql diff --git a/Cargo.lock b/Cargo.lock index d01517a41c..286fac3b92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4292,6 +4292,7 @@ dependencies = [ "heck", "humantime-serde", "itertools 0.14.0", + "libc", "log", "opentelemetry 0.30.0", "opentelemetry_sdk 0.30.0", diff --git a/docs/src/content/next/deploy.mdx b/docs/src/content/next/deploy.mdx index 098b1bc441..8eaf886fe8 100644 --- a/docs/src/content/next/deploy.mdx +++ b/docs/src/content/next/deploy.mdx @@ -44,7 +44,7 @@ See also: [configuration](https://github.com/golemcloud/golem/blob/main/golem-wo ### Worker Executor -[golem-worker-executor](https://github.com/golemcloud/golem/tree/main/golem-worker-executor) is responsible for running the [agents](/next/concepts/agents) that belong to assigned shards. The service uses Redis and Blob storage as data storage. +[golem-worker-executor](https://github.com/golemcloud/golem/tree/main/golem-worker-executor) is responsible for running the [agents](/next/concepts/agents) that belong to assigned shards. The service keeps each agent's oplog in **indexed storage** - PostgreSQL, SQLite or Redis - and uses key-value and blob storage alongside it. See also: [configuration](https://github.com/golemcloud/golem/blob/main/golem-worker-executor/config/worker-executor.toml), [environment variables](https://github.com/golemcloud/golem/blob/main/golem-worker-executor/config/worker-executor.sample.env), [docker image](https://hub.docker.com/r/golemservices/golem-worker-executor) @@ -57,6 +57,8 @@ See also: [configuration](https://github.com/golemcloud/golem/blob/main/golem-wo Every worker executor holds a **lease** on its shard assignment and renews it with the shard manager. A lease lasts `shard_lease_duration` (1m by default) and an executor renews at a third of the time it has left, so roughly every 20s. The shard manager refuses to start with a lease shorter than 30s and warns about one shorter than 1m: the deadline it allows each renewal call never drops below 10s, so a shorter lease cannot fit the three renewal attempts a lease is meant to get. Each renewal carries the epoch of every shard the executor holds; a shard's epoch is its ownership generation, advanced only when the shard moves to another executor and never by a renewal. A renewal whose claim does not match the shard manager's view is renewed all the same, and the response carries the shard manager's set, which the executor adopts exactly as it would an assignment push: pushes deliver a change at once, and renewals guarantee it arrives within a third of the lease even if the push was lost. The shard manager records a shard's new owner before it tells anybody about it, so a rebalance whose write is refused changes nothing at all, and every delivery carries the revision of the state it was read from. An executor ignores a delivery older than one it has already applied, so a push and a renewal that cross on the network cannot leave the older set in place. An executor that stops renewing loses its lease, and the shard manager reclaims and redistributes those shards on its next pass, which runs at least every third of `shard_lease_duration` even in a cluster where nothing else is happening; a crashed or unreachable executor's shards are therefore re-homed within about one lease duration plus one pass. The lease reaches the executor as the time remaining on it rather than as an absolute time, so the two machines' clocks are never compared and skew between them can neither lengthen nor shorten it. The executor anchors that time to the moment it sent the request the lease answers, so time the reply spends in flight comes off its copy of the lease and is never added to it, and its copy lapses no later than the shard manager's. An assignment push carries the shard set and its revision only: it never extends a lease. The lease is the one place the two machines' clocks must agree on a rate rather than on a value: an executor whose clock runs slow relative to the shard manager's keeps admitting for that fraction of a lease past the shard manager reclaiming it, which is milliseconds under ordinary `ntp` or `chrony` discipline and seconds while such a daemon is slewing off a large offset, so an executor should not be started until its clock is synchronised. An executor whose own copy of the lease has expired stops **admitting** new work for its shards and reports that sharding is not ready, which a worker service retries after refreshing its routing table - invocations already running are not interrupted. A graceful stop - including the `SIGTERM` an orchestrator sends to stop a pod - deregisters the executor instead of waiting for its lease to expire, so its shards move on the next pass. A shard manager restart or failover re-grants the lease of every executor that still answers the startup health check, so a restart never evicts a healthy cluster: only executors that fail that check are removed. An executor whose own copy of the lease lapsed during the outage resumes admitting work on its next successful renewal rather than on the push that follows the re-grant, and neither a re-grant nor a renewal ever moves an executor's deadline earlier, so a `shard_lease_duration` reduced across a restart takes effect only once the new length outruns the deadline each executor was last told. +The lease bounds how long a lost shard keeps being served, but it cannot stop an executor that has already lost one from finishing a write it had started. That is what **oplog fencing** is for. Each agent's oplog records the shard epoch allowed to write it, and every batch of entries is checked against that record inside the same database transaction that inserts them, so a refused write leaves nothing behind. An executor whose epoch is behind is turned away at the storage rather than discovering the problem later: that one agent stops there, and the worker service resumes it on the shard's new owner. The record only ever moves forward - an executor re-granted the shard at a higher epoch takes over, and one holding a stale epoch cannot claim it back - and an absent record fences too, because the record is written before an oplog's first entry and removed before its last. Fencing is enforced by the PostgreSQL and SQLite indexed storage backends. Redis and the in-memory backend cannot enforce it, so an executor configured with one of those **and** a shard manager refuses to start rather than run unprotected; set `indexed_storage.type` to `Postgres`, `Sqlite`, `KVStoreSqlite`, `MultiSqlite` or `KVStoreMultiSqlite`. Single-shard deployments and the debugging service are exempt, because nothing can take a shard away from them. During a rolling upgrade the fence is complete only once every executor runs the new code and each agent has been opened once under it. Ephemeral agents are not fenced: their oplogs are never replayed, so a duplicate there is a duplicated observability record rather than duplicated state. + Distributed mode talks to etcd over **plaintext HTTP** and without authentication. Every entry in `persistence.config.endpoints` must start with `http://`; a replica configured with anything else refuses to start. etcd therefore has to be reachable unauthenticated, either on a trusted network or through a TLS-terminating proxy running alongside the shard manager, with the endpoints pointing at that proxy. Give `persistence.config.endpoints` a **single load-balanced address** - the Kubernetes Service in front of the etcd cluster - rather than a list of members. With a member list the client keeps a member that is down in its rotation: reads retry through it and recover, but writes do not, and a leadership campaign can spend tens of seconds landing on the dead member before it gets anywhere. diff --git a/docs/src/content/next/operate/persistence.mdx b/docs/src/content/next/operate/persistence.mdx index 3c4aae5c4b..edc8fb7908 100644 --- a/docs/src/content/next/operate/persistence.mdx +++ b/docs/src/content/next/operate/persistence.mdx @@ -18,7 +18,7 @@ Currently Golem provides the following implementations, configurable through the | ------------------ | -------------------------- | | Blob storage | S3, file system, in-memory | | Key-values storage | Redis, in-memory | -| Indexed storage | Redis (streams), in-memory | +| Indexed storage | PostgreSQL, SQLite, Redis (streams), in-memory | ## Compilation cache diff --git a/golem-api-grpc/proto/golem/worker/invocation_session.proto b/golem-api-grpc/proto/golem/worker/invocation_session.proto index 6def18ca3b..2b26403651 100644 --- a/golem-api-grpc/proto/golem/worker/invocation_session.proto +++ b/golem-api-grpc/proto/golem/worker/invocation_session.proto @@ -238,6 +238,10 @@ enum InvocationRejectionReason { INVOCATION_REJECTION_REASON_INPUT_CONFLICT = 12; INVOCATION_REJECTION_REASON_INPUT_GAP = 13; INVOCATION_REJECTION_REASON_RESOURCE_EXHAUSTED = 14; + // The request reached an executor that does not own the agent's shard: a stale route, or an + // executor whose lease has lapsed. Not a refusal of the invocation - the caller retries it on + // the shard's owner after refreshing its routing table. + INVOCATION_REJECTION_REASON_SHARDING_NOT_READY = 15; } message InvocationRejected { diff --git a/golem-api-grpc/proto/golem/worker/raw_oplog.proto b/golem-api-grpc/proto/golem/worker/raw_oplog.proto index b34ed2de6a..b755d56fb2 100644 --- a/golem-api-grpc/proto/golem/worker/raw_oplog.proto +++ b/golem-api-grpc/proto/golem/worker/raw_oplog.proto @@ -200,6 +200,7 @@ message RawAgentInvocationStartedParameters { repeated string trace_states = 4; repeated RawSpanData invocation_context = 5; optional RawInvocationWalletPin wallet_pin = 6; + optional uint64 shard_epoch = 7; } message RawInvocationWalletPin { diff --git a/golem-common/src/base_model/oplog/mod.rs b/golem-common/src/base_model/oplog/mod.rs index d29e70295a..e1a73d3c31 100644 --- a/golem-common/src/base_model/oplog/mod.rs +++ b/golem-common/src/base_model/oplog/mod.rs @@ -185,7 +185,10 @@ oplog_entry! { } }, /// The agent has been invoked - #[desert(evolution(FieldAdded("wallet_pin", None::)))] + #[desert(evolution( + FieldAdded("wallet_pin", None::), + FieldAdded("shard_epoch", None::) + ))] AgentInvocationStarted { hint: false wit_raw_type: "raw-agent-invocation-started-parameters" @@ -197,6 +200,12 @@ oplog_entry! { trace_states: Vec, invocation_context: Vec, wallet_pin: Option, + /// The shard epoch this executor held for the agent's shard when the invocation + /// started. Raw only - it is a record of which ownership generation produced the + /// entry, for operators and oplog-processor plugins reading a divergence, not + /// something the agent's own history should expose. `None` for entries written + /// before the fence existed, and for oplogs opened without an epoch to assert. + shard_epoch: Option, } public { invocation: PublicAgentInvocation, diff --git a/golem-common/src/cache.rs b/golem-common/src/cache.rs index ca4f17a01c..703a1a027e 100644 --- a/golem-common/src/cache.rs +++ b/golem-common/src/cache.rs @@ -676,6 +676,36 @@ impl< self.state.items.contains_async(key).await } + /// Like [`Self::create_weak_remover`], but removes the cached value only if it satisfies + /// `predicate`: a remover tied to one particular value cannot evict a replacement that was + /// cached under the same key after it. Pending entries are never removed. + pub fn create_weak_remover_if( + &self, + key: K, + predicate: F, + ) -> impl FnOnce() + use + where + F: FnOnce(&V) -> bool, + { + let weak_state = Arc::downgrade(&self.state); + let name = self.name; + move || { + if let Some(state) = weak_state.upgrade() { + let removed = state + .items + .remove_if_sync(&key, |item| match item { + Item::Cached { value, .. } => predicate(value), + Item::Pending { .. } => false, + }) + .is_some(); + if removed { + let count = state.count.fetch_sub(1, Ordering::SeqCst); + record_cache_size(name, count.saturating_sub(1)); + } + } + } + } + pub fn create_weak_remover(&self, key: K) -> impl FnOnce() + use { let weak_state = Arc::downgrade(&self.state); let name = self.name; @@ -1066,6 +1096,28 @@ mod tests { assert!(!cache.contains_key(&1).await); } + #[test] + async fn weak_remover_if_leaves_a_value_it_was_not_created_for() { + let cache = test_cache("weak_remover_if"); + cache + .get_or_insert_simple(&1, || async { Ok(1u64) }) + .await + .unwrap(); + // Created for the first value, run after that value was replaced: the replacement stays. + let remover = cache.create_weak_remover_if(1, |v| *v == 1); + cache.remove(&1).await; + cache + .get_or_insert_simple(&1, || async { Ok(2u64) }) + .await + .unwrap(); + remover(); + assert!(cache.contains_key(&1).await); + + let remover = cache.create_weak_remover_if(1, |v| *v == 2); + remover(); + assert!(!cache.contains_key(&1).await); + } + #[test] async fn remove_if_cached_no_op_when_predicate_does_not_match() { let cache = test_cache("remove_if_cached_no_match"); diff --git a/golem-common/src/model/oplog/protobuf.rs b/golem-common/src/model/oplog/protobuf.rs index c1fb36632e..505aaae1e5 100644 --- a/golem-common/src/model/oplog/protobuf.rs +++ b/golem-common/src/model/oplog/protobuf.rs @@ -4050,6 +4050,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry trace_states, invocation_context, wallet_pin, + shard_epoch, .. } => Entry::AgentInvocationStarted(RawAgentInvocationStartedParameters { idempotency_key: Some(idempotency_key.into()), @@ -4061,6 +4062,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry .map(span_data_to_proto) .collect(), wallet_pin: wallet_pin.map(invocation_wallet_pin_to_proto), + shard_epoch, }), OplogEntry::AgentInvocationFinished { result, @@ -4635,6 +4637,7 @@ impl TryFrom for OplogEntry .wallet_pin .map(invocation_wallet_pin_from_proto) .transpose()?, + shard_epoch: p.shard_epoch, }) } Entry::AgentInvocationFinished(p) => { diff --git a/golem-common/src/model/oplog/tests.rs b/golem-common/src/model/oplog/tests.rs index 52b9c32756..352fede345 100644 --- a/golem-common/src/model/oplog/tests.rs +++ b/golem-common/src/model/oplog/tests.rs @@ -1248,6 +1248,75 @@ fn raw_snapshot_protobuf_roundtrip_preserves_active_cards() { } } +#[test] +fn shard_epoch_protobuf_roundtrip_and_legacy_default() { + fn started(shard_epoch: Option) -> OplogEntry { + OplogEntry::AgentInvocationStarted { + timestamp: Timestamp::now_utc().rounded(), + idempotency_key: IdempotencyKey::new("shard-epoch".to_string()), + payload: OplogPayload::Inline(Box::new(AgentInvocationPayload::AgentMethod { + method_name: "test".to_string(), + input: SchemaValue::Record { fields: Vec::new() }, + principal: Principal::anonymous(), + scope_card: None, + })), + trace_id: TraceId::generate(), + trace_states: Vec::new(), + invocation_context: Vec::new(), + wallet_pin: None, + shard_epoch, + } + } + + // Round-trips on disk, which is the channel that matters for replay. Compared field by + // field rather than whole-entry: an inline payload is re-represented as `SerializedInline` + // by the codec, so the decoded entry is deliberately not equal to the one that went in. + for expected in [Some(9u64), None] { + let entry = started(expected); + let bytes = crate::serialization::serialize(&entry).unwrap(); + let decoded: OplogEntry = crate::serialization::deserialize(&bytes).unwrap(); + match decoded { + OplogEntry::AgentInvocationStarted { shard_epoch, .. } => { + assert_eq!( + shard_epoch, expected, + "shard epoch after a desert round trip" + ); + } + other => panic!("expected raw invocation-started entry, got {other:?}"), + } + } + + // And on the raw protobuf, which is the oplog-processor-plugin channel. + let entry = started(Some(9)); + let mut raw_proto: golem_api_grpc::proto::golem::worker::RawOplogEntry = + entry.clone().try_into().unwrap(); + match OplogEntry::try_from(raw_proto.clone()).unwrap() { + OplogEntry::AgentInvocationStarted { shard_epoch, .. } => { + assert_eq!(shard_epoch, Some(9)); + } + other => panic!("expected raw invocation-started entry, got {other:?}"), + } + + // An entry from before the fence existed carries no epoch, and must decode as absent rather + // than as epoch zero - zero is a real epoch, held by the first owner of every shard. + if let Some( + golem_api_grpc::proto::golem::worker::raw_oplog_entry::Entry::AgentInvocationStarted( + params, + ), + ) = &mut raw_proto.entry + { + params.shard_epoch = None; + } else { + panic!("expected raw invocation-started protobuf entry"); + } + match OplogEntry::try_from(raw_proto).unwrap() { + OplogEntry::AgentInvocationStarted { shard_epoch, .. } => { + assert_eq!(shard_epoch, None); + } + other => panic!("expected raw invocation-started entry, got {other:?}"), + } +} + #[test] fn invocation_wallet_pin_protobuf_roundtrip_and_legacy_defaults() { let pinned_card_ids = vec![CardId::new(), CardId::new()]; @@ -1273,6 +1342,7 @@ fn invocation_wallet_pin_protobuf_roundtrip_and_legacy_defaults() { pinned_card_ids: pinned_card_ids.clone(), scope_card_id: Some(scope_card_id), }), + shard_epoch: None, }; let mut raw_proto: golem_api_grpc::proto::golem::worker::RawOplogEntry = diff --git a/golem-debugging-service/src/oplog/debug_oplog.rs b/golem-debugging-service/src/oplog/debug_oplog.rs index 292c87f9cd..dbb56a30ad 100644 --- a/golem-debugging-service/src/oplog/debug_oplog.rs +++ b/golem-debugging-service/src/oplog/debug_oplog.rs @@ -79,14 +79,18 @@ impl Oplog for DebugOplog { // live-repair an incomplete durable call // (`DebugContext::ALLOW_LIVE_REPAIR_OF_INCOMPLETE_DURABLE_CALLS` is `false`), so no repaired // `Start`/`End` pair is ever created against a `NONE` index during replay. - async fn add(&self, _entry: OplogEntry) -> OplogIndex { - OplogIndex::NONE + async fn add( + &self, + _entry: OplogEntry, + ) -> Result { + Ok(OplogIndex::NONE) } async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, golem_worker_executor::services::oplog::OplogError> + { Ok(make_batch(OplogIndex::NONE) .into_iter() .map(|record| (OplogIndex::NONE, record.into_inline_entry())) @@ -94,7 +98,7 @@ impl Oplog for DebugOplog { } fn enqueue_add(&self, _entry: OplogEntry) -> OplogAddReceipt { - Box::pin(async { OplogIndex::NONE }) + Box::pin(async { Ok(OplogIndex::NONE) }) } // Mirrors `add`: a debugging session never writes to the oplog, so both entries are built (to @@ -103,9 +107,9 @@ impl Oplog for DebugOplog { &self, _start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), golem_worker_executor::services::oplog::OplogError> { let _second = make_second(OplogIndex::NONE); - (OplogIndex::NONE, OplogIndex::NONE) + Ok((OplogIndex::NONE, OplogIndex::NONE)) } // Mirrors `add`: a debugging session never writes to the oplog, so this builds the `Start` (to @@ -114,9 +118,9 @@ impl Oplog for DebugOplog { &self, serialized_request: Vec, build_start: Box Result + Send>, - ) -> Result { + ) -> Result { let entry = build_start(RawOplogPayload::SerializedInline(serialized_request))?; - let index = self.add(entry.clone()).await; + let index = self.add(entry.clone()).await?; Ok(OrderedOplogStart { index, entry, @@ -127,7 +131,7 @@ impl Oplog for DebugOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let (serialized_request, build_start) = build_request(OplogIndex::NONE)?; self.add_start_with_reserved_raw_payload(serialized_request, build_start) .await @@ -138,8 +142,12 @@ impl Oplog for DebugOplog { } // There is no need to commit anything to the indexed storage - async fn commit(&self, _level: CommitLevel) -> BTreeMap { - BTreeMap::new() + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, golem_worker_executor::services::oplog::OplogError> + { + Ok(BTreeMap::new()) } // Current Oplog Index acts as the Replay Target diff --git a/golem-debugging-service/src/oplog/debug_oplog_constructor.rs b/golem-debugging-service/src/oplog/debug_oplog_constructor.rs index 93b5410e70..512c2aaa4c 100644 --- a/golem-debugging-service/src/oplog/debug_oplog_constructor.rs +++ b/golem-debugging-service/src/oplog/debug_oplog_constructor.rs @@ -74,6 +74,9 @@ impl OplogConstructor for CreateDebugOplogConstructor { self.initial_worker_metadata.clone(), self.last_known_status.clone(), self.execution_status.clone(), + // A debugging session discards every write, so it asserts no epoch and + // never touches the agent's ownership record. + None, ) .await } else { @@ -85,6 +88,9 @@ impl OplogConstructor for CreateDebugOplogConstructor { self.initial_worker_metadata.clone(), self.last_known_status.clone(), self.execution_status.clone(), + // A debugging session discards every write, so it asserts no epoch and + // never touches the agent's ownership record. + None, ) .await }; diff --git a/golem-debugging-service/src/oplog/debug_oplog_service.rs b/golem-debugging-service/src/oplog/debug_oplog_service.rs index 03a9143a56..86ffd4b63e 100644 --- a/golem-debugging-service/src/oplog/debug_oplog_service.rs +++ b/golem-debugging-service/src/oplog/debug_oplog_service.rs @@ -69,6 +69,7 @@ impl OplogService for DebugOplogService { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { panic!("Cannot create a new oplog when debugging") } @@ -81,6 +82,7 @@ impl OplogService for DebugOplogService { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { panic!("Cannot create a new oplog when debugging") } @@ -93,6 +95,7 @@ impl OplogService for DebugOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( diff --git a/golem-debugging-service/src/services/debug_service.rs b/golem-debugging-service/src/services/debug_service.rs index 2fbf10cc71..d74ebf36db 100644 --- a/golem-debugging-service/src/services/debug_service.rs +++ b/golem-debugging-service/src/services/debug_service.rs @@ -1223,7 +1223,10 @@ mod tests { #[async_trait] impl Oplog for SeqOplog { - async fn add(&self, _entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + _entry: OplogEntry, + ) -> Result { unimplemented!() } @@ -1238,7 +1241,8 @@ mod tests { &self, _start: OplogEntry, _make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), golem_worker_executor::services::oplog::OplogError> + { unimplemented!() } @@ -1246,14 +1250,20 @@ mod tests { &self, _serialized_request: Vec, _build_start: Box Result + Send>, - ) -> Result { + ) -> Result< + golem_worker_executor::services::oplog::OrderedOplogStart, + golem_worker_executor::services::oplog::OplogError, + > { unimplemented!() } async fn add_start_with_indexed_reserved_raw_payload( &self, _build_request: golem_worker_executor::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result< + golem_worker_executor::services::oplog::OrderedOplogStart, + golem_worker_executor::services::oplog::OplogError, + > { unimplemented!() } @@ -1261,7 +1271,13 @@ mod tests { unimplemented!() } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + _level: CommitLevel, + ) -> Result< + BTreeMap, + golem_worker_executor::services::oplog::OplogError, + > { unimplemented!() } @@ -1335,7 +1351,10 @@ mod tests { #[async_trait] impl Oplog for TestOplog { - async fn add(&self, _entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + _entry: OplogEntry, + ) -> Result { unimplemented!() } @@ -1350,7 +1369,8 @@ mod tests { &self, _start: OplogEntry, _make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), golem_worker_executor::services::oplog::OplogError> + { unimplemented!() } @@ -1358,14 +1378,20 @@ mod tests { &self, _serialized_request: Vec, _build_start: Box Result + Send>, - ) -> Result { + ) -> Result< + golem_worker_executor::services::oplog::OrderedOplogStart, + golem_worker_executor::services::oplog::OplogError, + > { unimplemented!() } async fn add_start_with_indexed_reserved_raw_payload( &self, _build_request: golem_worker_executor::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result< + golem_worker_executor::services::oplog::OrderedOplogStart, + golem_worker_executor::services::oplog::OplogError, + > { unimplemented!() } @@ -1373,7 +1399,13 @@ mod tests { unimplemented!() } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + _level: CommitLevel, + ) -> Result< + BTreeMap, + golem_worker_executor::services::oplog::OplogError, + > { unimplemented!() } diff --git a/golem-service-base/src/error/worker_executor.rs b/golem-service-base/src/error/worker_executor.rs index 06be0b7f8c..2c51f4b4a3 100644 --- a/golem-service-base/src/error/worker_executor.rs +++ b/golem-service-base/src/error/worker_executor.rs @@ -131,6 +131,16 @@ pub enum WorkerExecutorError { PermissionDenied { details: String, }, + /// A write to the agent's oplog was refused by the storage because the shard epoch this + /// executor asserted is behind the one recorded for the oplog: another executor owns the + /// shard now. Typed so the invocation loop can stop the agent cleanly instead of treating + /// it as a runtime failure to retry; crosses the wire as `ShardingNotReady`, which the + /// worker service already answers by refreshing its routing table and retrying. + OplogFenced { + agent_id: AgentId, + expected_epoch: u64, + actual_epoch: Option, + }, } impl WorkerExecutorError { @@ -184,6 +194,14 @@ impl WorkerExecutorError { } } + pub fn oplog_fenced(agent_id: AgentId, expected_epoch: u64, actual_epoch: Option) -> Self { + Self::OplogFenced { + agent_id, + expected_epoch, + actual_epoch, + } + } + pub fn invalid_shard_id(shard_id: ShardId, shard_ids: HashSet) -> Self { Self::InvalidShardId { shard_id, @@ -336,6 +354,22 @@ impl Display for WorkerExecutorError { Self::PermissionDenied { details } => { write!(f, "Permission denied: {details}") } + Self::OplogFenced { + agent_id, + expected_epoch, + actual_epoch, + } => match actual_epoch { + Some(actual) => write!( + f, + "Oplog write for {agent_id} fenced: this executor asserted shard epoch \ + {expected_epoch}, the stored epoch is {actual}" + ), + None => write!( + f, + "Oplog write for {agent_id} fenced: this executor asserted shard epoch \ + {expected_epoch}, but no epoch is stored for the oplog" + ), + }, } } } @@ -380,6 +414,7 @@ impl Error for WorkerExecutorError { Self::FileSystemError { .. } => "File system error", Self::ReadOnlyViolation { .. } => "Read-only agent method attempted a side effect", Self::PermissionDenied { .. } => "Permission denied", + Self::OplogFenced { .. } => "Oplog write fenced: the shard has a new owner", } } } @@ -416,6 +451,7 @@ impl ApiErrorDetails for WorkerExecutorError { Self::FileSystemError { .. } => "FileSystemError", Self::ReadOnlyViolation { .. } => "ReadOnlyViolation", Self::PermissionDenied { .. } => "PermissionDenied", + Self::OplogFenced { .. } => "OplogFenced", } } @@ -428,6 +464,7 @@ impl ApiErrorDetails for WorkerExecutorError { | Self::PromiseAlreadyCompleted { .. } | Self::Interrupted { .. } | Self::InvalidShardId { .. } + | Self::OplogFenced { .. } | Self::ComponentNotFound { .. } => true, Self::InvalidRequest { .. } | Self::AgentCreationFailed { .. } @@ -796,6 +833,15 @@ impl From for golem::worker::v1::WorkerExecutionError { ), ), }, + // The client cannot act on the epochs; what it can do is what it does for a lapsed + // lease - refresh its routing table and retry on the owner. + WorkerExecutorError::OplogFenced { .. } => Self { + error: Some( + golem::worker::v1::worker_execution_error::Error::ShardingNotReady( + golem::worker::v1::ShardingNotReady {}, + ), + ), + }, } } } @@ -1090,6 +1136,11 @@ pub enum InterruptKind { Restart, Suspend(Timestamp), Jump, + /// This executor no longer owns the agent's shard. Terminal here: the agent is stopped + /// without writing to its oplog or its status, dropped from the executor, and left for the + /// worker service to resume on the shard's owner. Never a restart in place - that would + /// reopen the oplog with the same stale epoch. + ShardLost, } impl Display for InterruptKind { @@ -1099,6 +1150,9 @@ impl Display for InterruptKind { InterruptKind::Restart => write!(f, "Simulated crash via the Golem API"), InterruptKind::Suspend(_) => write!(f, "Suspended"), InterruptKind::Jump => write!(f, "Jumping back in time"), + InterruptKind::ShardLost => { + write!(f, "This executor no longer owns the agent's shard") + } } } } diff --git a/golem-shard-manager/src/sharding/model.rs b/golem-shard-manager/src/sharding/model.rs index 873f4bf1d2..f1e17fc0c9 100644 --- a/golem-shard-manager/src/sharding/model.rs +++ b/golem-shard-manager/src/sharding/model.rs @@ -475,6 +475,72 @@ impl ShardLeaseState { .collect() } + /// Raises the recorded epoch of every shard `claimed` names to at least the claimed value, + /// and reports the shards that moved. + /// + /// A claim can only be ahead of what is recorded here if this state regressed, because an + /// executor is never told an epoch that was not written first: a rebalance is stored before + /// any of it is sent, and a grant is read off the state that is about to be persisted. So in + /// ordinary operation this is a no-op, and a non-empty result means the store lost history - + /// it was wiped, restored from a backup, or replaced. + /// + /// That case has to be repaired, because [`Self::shard_epochs`] is the only record of it. A + /// fresh state starts the epochs at zero while the executors, and the oplog rows their writes + /// are fenced against, still hold higher ones; since an epoch only climbs when a shard changes + /// owner, the fence would refuse those agents for good. The renewal carries the executor's + /// whole set, so it is the one moment the cluster can tell the manager what it forgot. + /// + /// Only `executor_id`'s own shards are repaired from its claim; a claim on a shard the + /// manager has given to somebody else is corrected by the grant, never adopted. + /// + /// The assignment moves with the high-water and never apart from it - [`Self::check_invariants`] + /// requires the two to agree - and the corrected grant the renewal returns is what tells the + /// current owner its new epoch. The value only ever climbs, so this cannot walk an epoch back + /// to one a stale writer still holds. + pub fn raise_epoch_floor( + &mut self, + executor_id: ExecutorId, + claimed: &BTreeMap, + ) -> Vec { + let mut raised = Vec::new(); + for (shard_id, claimed_epoch) in claimed { + // A claim naming a shard outside the current count is stale in a way this cannot + // repair, and recording it would break the invariants. + if !self.contains_shard(*shard_id) { + continue; + } + // A claim on a shard the manager has given to somebody else is an executor that + // missed a push, not evidence about that shard's epoch. Stamping the claim onto the + // owner's entry would put two live executors on one `(shard, epoch)` - the pair the + // fence cannot tell apart, which is the whole point of the epoch. The grant corrects + // that claim instead, like any other stale entry. An unassigned shard has no entry to + // corrupt, and raising its high-water only makes the next mint start above the epoch + // the claim proves is already in use. + if self + .shard_assignments + .get(shard_id) + .is_some_and(|entry| entry.executor_id != executor_id) + { + continue; + } + // Against the high-water, so the floor only ever climbs: a value below one this shard + // has already reached is not evidence of anything. + if self + .shard_epochs + .get(shard_id) + .is_some_and(|recorded| recorded >= claimed_epoch) + { + continue; + } + self.shard_epochs.insert(*shard_id, *claimed_epoch); + if let Some(entry) = self.shard_assignments.get_mut(shard_id) { + entry.epoch = *claimed_epoch; + } + raised.push(*shard_id); + } + raised + } + /// The ownership epoch `shard_id` takes when it is assigned to `executor_id`: unchanged while /// the owner stays the same, one past the highest epoch ever recorded for that shard when the /// owner changes. @@ -1139,6 +1205,139 @@ mod tests { assert!(shard_state.check_invariants().is_ok()); } + #[test] + fn a_renewal_raises_epochs_the_state_lost() { + // The store was wiped and rebuilt: executor 1 re-registered and was handed its shards + // back, but from a state with no memory of what they were worth. + let mut shard_state = shard_state_with(4, &[(1, 1, &[0, 1])]); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); + assert_eq!(shard_state.epoch_for_shard(shard(1)), Some(ShardEpoch(0))); + + // The executor still holds - and its oplog rows are still fenced against - the epochs it + // was granted before the wipe. + let claimed = BTreeMap::from([(shard(0), ShardEpoch(5)), (shard(1), ShardEpoch(3))]); + assert_eq!( + shard_state.raise_epoch_floor(executor(1), &claimed), + vec![shard(0), shard(1)] + ); + + // Both halves move together, so the state stays consistent ... + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(5))); + assert_eq!(shard_state.epoch_for_shard(shard(1)), Some(ShardEpoch(3))); + assert_eq!( + shard_state.shard_epochs.get(&shard(0)), + Some(&ShardEpoch(5)) + ); + assert_eq!( + shard_state.shard_epochs.get(&shard(1)), + Some(&ShardEpoch(3)) + ); + assert!(shard_state.check_invariants().is_ok()); + + // ... and the next owner change mints above the restored high-water rather than + // re-issuing an epoch some oplog row already holds. + shard_state.add_executor(executor(2), addr(2), None, t0(), TTL); + assert_eq!( + shard_state.assign_shard(executor(2), shard(0)), + ShardEpoch(6) + ); + assert!(shard_state.check_invariants().is_ok()); + } + + #[test] + fn raise_epoch_floor_only_ever_climbs() { + let mut shard_state = shard_state_with(4, &[(1, 1, &[0])]); + shard_state.remove_executor(executor(1)); + shard_state.add_executor(executor(2), addr(2), None, t0(), TTL); + shard_state.assign_shard(executor(2), shard(0)); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); + + // The ordinary case: a claim from an executor that missed a push is BEHIND the record, + // and must not walk the epoch back to one the previous owner still holds. + let stale = BTreeMap::from([(shard(0), ShardEpoch(0))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &stale) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); + + // A claim that matches is not a regression either. + let matching = BTreeMap::from([(shard(0), ShardEpoch(1))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &matching) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); + + // A claim naming a shard outside the cluster's count is ignored rather than recorded: + // an epoch there would fail the invariants. + let out_of_range = BTreeMap::from([(shard(9), ShardEpoch(7))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &out_of_range) + .is_empty() + ); + assert!(!shard_state.shard_epochs.contains_key(&shard(9))); + + // An unassigned shard can still have its floor restored, so the next assignment mints + // above it. + let unassigned = BTreeMap::from([(shard(2), ShardEpoch(4))]); + assert_eq!( + shard_state.raise_epoch_floor(executor(2), &unassigned), + vec![shard(2)] + ); + assert_eq!( + shard_state.shard_epochs.get(&shard(2)), + Some(&ShardEpoch(4)) + ); + assert_eq!(shard_state.epoch_for_shard(shard(2)), None); + assert!(shard_state.check_invariants().is_ok()); + assert_eq!( + shard_state.assign_shard(executor(2), shard(2)), + ShardEpoch(5) + ); + } + + #[test] + fn a_claim_on_another_executors_shard_never_raises_it() { + // Executor 1 holds shard 0; executor 2 missed the push that took it away and still claims + // it, at an epoch above the record. Adopting that would stamp executor 2's epoch onto + // executor 1's assignment, leaving both of them live on `(shard 0, epoch 9)` - which is + // exactly the pair the oplog fence cannot separate. + let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[1])]); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); + + let poaching = BTreeMap::from([(shard(0), ShardEpoch(9))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &poaching) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); + assert_eq!( + shard_state + .shard_assignments + .get(&shard(0)) + .map(|e| e.executor_id), + Some(executor(1)), + "the owner was not changed either" + ); + assert_eq!( + shard_state.shard_epochs.get(&shard(0)), + Some(&ShardEpoch(0)) + ); + + // The owner's own claim at the same epoch is still repaired. + assert_eq!( + shard_state.raise_epoch_floor(executor(1), &poaching), + vec![shard(0)] + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(9))); + assert!(shard_state.check_invariants().is_ok()); + } + #[test] fn unassign_shard_is_guarded_by_owner() { let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[])]); diff --git a/golem-shard-manager/src/sharding/shard_management.rs b/golem-shard-manager/src/sharding/shard_management.rs index 2608f584d0..78b1deb77f 100644 --- a/golem-shard-manager/src/sharding/shard_management.rs +++ b/golem-shard-manager/src/sharding/shard_management.rs @@ -353,6 +353,18 @@ impl ShardManagement { ); } + // Ahead of the renewal, so the grant read below carries the repaired epochs. This + // only ever fires when the stored state is behind the cluster it is managing. + let raised = shard_state.raise_epoch_floor(executor_id, &claimed); + if !raised.is_empty() { + warn!( + executor_id = %executor_id, + raised_shards = raised.iter().join(", "), + "Shard lease claim carried epochs ahead of the stored state; raising them. \ + The shard state has lost history - it was wiped, restored or replaced" + ); + } + if !shard_state.renew_lease(executor_id, now, lease_ttl) { return Err(ShardManagerError::Internal(format!( "executor {executor_id} holds no lease right after it was found" diff --git a/golem-shard-manager/tests/shard_management.rs b/golem-shard-manager/tests/shard_management.rs index 3ccae105e2..e34db60f36 100644 --- a/golem-shard-manager/tests/shard_management.rs +++ b/golem-shard-manager/tests/shard_management.rs @@ -2046,9 +2046,12 @@ async fn a_mismatched_claim_is_renewed_and_corrected() { "got {err:?}" ); - // a wrong epoch, alongside a claim entry that is perfectly valid + // a wrong epoch on a shard that belongs to somebody else, alongside claim entries that are + // perfectly valid. Corrected and never adopted: an epoch stamped onto another executor's + // assignment would leave both of them live on one `(shard, epoch)`, which is the one pair the + // oplog fence cannot separate. let mut wrong_epoch = truth.clone(); - wrong_epoch.insert(ShardId::new(1), ShardEpoch(7)); + wrong_epoch.insert(ShardId::new(2), ShardEpoch(7)); let expiry_before = expiry_of(&persistence.latest().await, executor(1)); let grant = shard_management .renew_shard_lease(executor(1), wrong_epoch) @@ -2059,6 +2062,11 @@ async fn a_mismatched_claim_is_renewed_and_corrected() { "the grant is the manager's set, not the claim" ); assert!(grant.expires_at > expiry_before, "the lease was extended"); + assert_eq!( + persistence.latest().await.epoch_for_shard(ShardId::new(2)), + Some(ShardEpoch(0)), + "the claim moved another executor's shard" + ); // a shard that belongs to another executor let moved = BTreeMap::from([(ShardId::new(2), ShardEpoch(0))]); @@ -2094,6 +2102,49 @@ async fn a_mismatched_claim_is_renewed_and_corrected() { join_set.abort_all(); } +#[test] +// The one case where a claim moves the manager's state rather than being corrected by it. An +// executor is never told an epoch the store did not hold first, so a claim ahead of the record is +// only possible when the store lost history - wiped, restored from a backup, or replaced. The +// executors' oplog rows are still fenced against the epochs they were granted before the loss, so +// a manager that went on granting from a lower floor would have every one of their writes refused +// for good. The renewal is the one moment the cluster can tell the manager what it forgot. +async fn a_claim_ahead_of_the_record_raises_the_managers_floor() { + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + + let before = persistence.latest().await; + assert_eq!(before.epoch_for_shard(ShardId::new(1)), Some(ShardEpoch(0))); + + let mut ahead = claim_of(&before, executor(1)); + ahead.insert(ShardId::new(1), ShardEpoch(7)); + let grant = shard_management + .renew_shard_lease(executor(1), ahead.clone()) + .await + .expect("a claim ahead of the record is renewed, not refused"); + + // The grant is still read off the manager's state - that state was repaired first, so the + // owner is told the epoch it already holds instead of one its oplog rows would refuse. + assert_eq!(grant.shard_epochs, ahead); + + let after = persistence.latest().await; + assert_eq!(after.epoch_for_shard(ShardId::new(1)), Some(ShardEpoch(7))); + assert_eq!(claim_of(&after, executor(1)), ahead); + assert_eq!( + after.shards_for_executor(executor(1)), + Some(shard_ids(&[0, 1])), + "repairing an epoch moved a shard" + ); + assert_eq!( + after.shards_for_executor(executor(2)), + Some(shard_ids(&[2, 3])), + "repairing an epoch disturbed another executor" + ); + + join_set.abort_all(); +} + #[test] // Every delivery carries the revision of the persisted state that holds its set, so the executor // can order a push and a renewal response that cross on the network. The grant is read off the diff --git a/golem-test-framework/Cargo.toml b/golem-test-framework/Cargo.toml index 807d2c6586..450959f9b7 100644 --- a/golem-test-framework/Cargo.toml +++ b/golem-test-framework/Cargo.toml @@ -61,5 +61,8 @@ url = { workspace = true } uuid = { workspace = true } wasm-metadata = { workspace = true } +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } + [features] default = [] diff --git a/golem-test-framework/src/components/mod.rs b/golem-test-framework/src/components/mod.rs index 034b1788c5..a36fc41300 100644 --- a/golem-test-framework/src/components/mod.rs +++ b/golem-test-framework/src/components/mod.rs @@ -248,6 +248,28 @@ fn check_child_process_alive(child: &mut Child, name: &str) { } } +/// One gRPC health check with a bounded wait: whether the service at `host:grpc_port` answers +/// `Serving` right now. Unlike a process-liveness check this also fails for a process that has +/// stopped serving but not yet exited - one aborting while the OS writes its crash report. +pub async fn is_serving_grpc(host: &str, grpc_port: u16, timeout: Duration) -> bool { + let probe = async { + let mut client = + golem_api_grpc::proto::grpc::health::v1::health_client::HealthClient::connect(format!( + "http://{host}:{grpc_port}" + )) + .await + .ok()?; + let response = client + .check(HealthCheckRequest { + service: "".to_string(), + }) + .await + .ok()?; + Some(response.into_inner().status == ServingStatus::Serving as i32) + }; + matches!(tokio::time::timeout(timeout, probe).await, Ok(Some(true))) +} + pub async fn wait_for_startup_grpc( host: &str, grpc_port: u16, diff --git a/golem-test-framework/src/components/shard_manager/mod.rs b/golem-test-framework/src/components/shard_manager/mod.rs index 565043fe46..6a4ee78ab5 100644 --- a/golem-test-framework/src/components/shard_manager/mod.rs +++ b/golem-test-framework/src/components/shard_manager/mod.rs @@ -88,6 +88,7 @@ async fn wait_for_startup( async fn env_vars( number_of_shards_override: Option, + shard_lease_duration_override: Option, http_port: u16, grpc_port: u16, rdb: &Arc, @@ -128,5 +129,12 @@ async fn env_vars( builder = builder.with("GOLEM__NUMBER_OF_SHARDS", number_of_shards.to_string()); } + if let Some(shard_lease_duration) = shard_lease_duration_override { + builder = builder.with( + "GOLEM__SHARD_LEASE_DURATION", + format!("{}ms", shard_lease_duration.as_millis()), + ); + } + builder.build() } diff --git a/golem-test-framework/src/components/shard_manager/spawned.rs b/golem-test-framework/src/components/shard_manager/spawned.rs index e0edef24e7..6b5e47ba0f 100644 --- a/golem-test-framework/src/components/shard_manager/spawned.rs +++ b/golem-test-framework/src/components/shard_manager/spawned.rs @@ -29,6 +29,7 @@ pub struct SpawnedShardManager { http_port: u16, grpc_port: u16, number_of_shards_override: std::sync::RwLock>, + shard_lease_duration_override: Option, child: Arc>>, logger: Arc>>, executable: PathBuf, @@ -46,6 +47,7 @@ impl SpawnedShardManager { executable: &Path, working_directory: &Path, number_of_shards_override: Option, + shard_lease_duration_override: Option, http_port: u16, grpc_port: u16, rdb: Arc, @@ -65,6 +67,7 @@ impl SpawnedShardManager { executable, working_directory, number_of_shards_override, + shard_lease_duration_override, http_port, grpc_port, &rdb, @@ -80,6 +83,7 @@ impl SpawnedShardManager { http_port, grpc_port, number_of_shards_override: std::sync::RwLock::new(number_of_shards_override), + shard_lease_duration_override, child: Arc::new(Mutex::new(Some(child))), logger: Arc::new(Mutex::new(Some(logger))), executable: executable.to_path_buf(), @@ -97,6 +101,7 @@ impl SpawnedShardManager { executable: &Path, working_directory: &Path, number_of_shards_override: Option, + shard_lease_duration_override: Option, http_port: u16, grpc_port: u16, rdb: &Arc, @@ -111,6 +116,7 @@ impl SpawnedShardManager { .envs( super::env_vars( number_of_shards_override, + shard_lease_duration_override, http_port, grpc_port, rdb, @@ -181,6 +187,7 @@ impl ShardManager for SpawnedShardManager { &self.executable, &self.working_directory, number_of_shards_override, + self.shard_lease_duration_override, self.http_port, self.grpc_port, &self.rdb, diff --git a/golem-test-framework/src/components/worker_executor/mod.rs b/golem-test-framework/src/components/worker_executor/mod.rs index 569f3202da..3fd78c673d 100644 --- a/golem-test-framework/src/components/worker_executor/mod.rs +++ b/golem-test-framework/src/components/worker_executor/mod.rs @@ -53,6 +53,24 @@ pub trait WorkerExecutor: Send + Sync { ); } + /// Freezes this worker executor's process in place (SIGSTOP) without killing it: it keeps its + /// sockets, its memory and every lease it believes it holds, but answers nothing until + /// [`WorkerExecutor::resume`]. This is how an executor is made to look dead to the rest of + /// the cluster while it still thinks it owns its shards, which a kill cannot do. + /// + /// Default implementation panics: only `SpawnedWorkerExecutor` owns a process to freeze. + async fn pause(&self) { + panic!("WorkerExecutor::pause is only supported by SpawnedWorkerExecutor"); + } + + /// Thaws a process frozen by [`WorkerExecutor::pause`] (SIGCONT). It carries on from exactly + /// where it stopped. + /// + /// Default implementation panics: only `SpawnedWorkerExecutor` owns a process to thaw. + async fn resume(&self) { + panic!("WorkerExecutor::resume is only supported by SpawnedWorkerExecutor"); + } + async fn is_running(&self) -> bool; } diff --git a/golem-test-framework/src/components/worker_executor/spawned.rs b/golem-test-framework/src/components/worker_executor/spawned.rs index b472c99c59..e6afd96f69 100644 --- a/golem-test-framework/src/components/worker_executor/spawned.rs +++ b/golem-test-framework/src/components/worker_executor/spawned.rs @@ -185,6 +185,28 @@ impl SpawnedWorkerExecutor { } let _logger = self.logger.lock().unwrap().take(); } + + #[cfg(unix)] + fn signal_child(&self, signal: libc::c_int, action: &str) { + let child = self.child.lock().unwrap(); + let child = child.as_ref().unwrap_or_else(|| { + panic!( + "Cannot {action} golem-worker-executor {}: it is not running", + self.grpc_port + ) + }); + let pid = libc::pid_t::try_from(child.id()).expect("child pid does not fit into pid_t"); + // SAFETY: `kill` has no memory-safety preconditions. The pid belongs to a child this struct + // spawned and has not reaped, so it cannot have been reused by another process. + let result = unsafe { libc::kill(pid, signal) }; + assert_eq!( + result, + 0, + "Failed to {action} golem-worker-executor {}: {}", + self.grpc_port, + std::io::Error::last_os_error() + ); + } } #[async_trait] @@ -257,6 +279,18 @@ impl WorkerExecutor for SpawnedWorkerExecutor { false } } + + #[cfg(unix)] + async fn pause(&self) { + info!("Pausing golem-worker-executor {}", self.grpc_port); + self.signal_child(libc::SIGSTOP, "pause"); + } + + #[cfg(unix)] + async fn resume(&self) { + info!("Resuming golem-worker-executor {}", self.grpc_port); + self.signal_child(libc::SIGCONT, "resume"); + } } impl Drop for SpawnedWorkerExecutor { diff --git a/golem-test-framework/src/components/worker_executor_cluster/mod.rs b/golem-test-framework/src/components/worker_executor_cluster/mod.rs index 1ca1014dce..4d0e64da8d 100644 --- a/golem-test-framework/src/components/worker_executor_cluster/mod.rs +++ b/golem-test-framework/src/components/worker_executor_cluster/mod.rs @@ -46,6 +46,21 @@ pub trait WorkerExecutorCluster: Send + Sync { async fn stop(&self, index: usize); async fn start(&self, index: usize); + /// Freezes the executor at `index` in place; see [`WorkerExecutor::pause`]. It still counts + /// as started: only a stop takes it out of the cluster. + /// + /// Default implementation panics: only `SpawnedWorkerExecutorCluster` owns the processes. + async fn pause(&self, _index: usize) { + panic!("WorkerExecutorCluster::pause is only supported by SpawnedWorkerExecutorCluster"); + } + + /// Thaws the executor at `index`; see [`WorkerExecutor::resume`]. + /// + /// Default implementation panics: only `SpawnedWorkerExecutorCluster` owns the processes. + async fn resume(&self, _index: usize) { + panic!("WorkerExecutorCluster::resume is only supported by SpawnedWorkerExecutorCluster"); + } + fn to_vec(&self) -> Vec>; async fn stopped_indices(&self) -> Vec; diff --git a/golem-test-framework/src/components/worker_executor_cluster/spawned.rs b/golem-test-framework/src/components/worker_executor_cluster/spawned.rs index 684b92a6c3..00c2b85aad 100644 --- a/golem-test-framework/src/components/worker_executor_cluster/spawned.rs +++ b/golem-test-framework/src/components/worker_executor_cluster/spawned.rs @@ -194,6 +194,14 @@ impl WorkerExecutorCluster for SpawnedWorkerExecutorCluster { } } + async fn pause(&self, index: usize) { + self.worker_executors[index].pause().await; + } + + async fn resume(&self, index: usize) { + self.worker_executors[index].resume().await; + } + fn to_vec(&self) -> Vec> { self.worker_executors.to_vec() } diff --git a/golem-test-framework/src/config/benchmark.rs b/golem-test-framework/src/config/benchmark.rs index 864f215990..61aacb58ad 100644 --- a/golem-test-framework/src/config/benchmark.rs +++ b/golem-test-framework/src/config/benchmark.rs @@ -430,6 +430,7 @@ impl BenchmarkTestDependencies { &build_root.join("golem-shard-manager"), &workspace_root.join("golem-shard-manager"), None, + None, shard_manager_http_port, shard_manager_grpc_port, rdb.clone(), diff --git a/golem-test-framework/src/config/env.rs b/golem-test-framework/src/config/env.rs index 608efe7fd4..e53071c2a1 100644 --- a/golem-test-framework/src/config/env.rs +++ b/golem-test-framework/src/config/env.rs @@ -93,6 +93,9 @@ pub struct EnvBasedTestDependenciesConfig { pub worker_executor_cluster_size: usize, pub environment_state_cache_capacity: Option, pub number_of_shards_override: Option, + /// The shard manager's `shard_lease_duration`, for tests that have to watch a lease lapse + /// without waiting out the default. `None` keeps the shard manager's own default. + pub shard_lease_duration_override: Option, pub oplog_archive_interval: Option, pub shared_client: bool, pub db_type: DbType, @@ -237,6 +240,7 @@ impl Default for EnvBasedTestDependenciesConfig { worker_executor_cluster_size: 4, environment_state_cache_capacity: None, number_of_shards_override: None, + shard_lease_duration_override: None, oplog_archive_interval: None, shared_client: false, db_type: DbType::Postgres, @@ -345,6 +349,7 @@ impl EnvBasedTestDependencies { &config.debug_targets_dirs().join("golem-shard-manager"), &config.golem_repo_root.join("golem-shard-manager"), config.number_of_shards_override, + config.shard_lease_duration_override, 9021, 9020, rdb, @@ -905,9 +910,15 @@ pub trait WorkerExecutorClusterControl { async fn restart_all_with_env_vars(&self, vars: Vec<(String, String)>); async fn stop(&self, idx: u16); async fn start(&self, idx: u16); + async fn pause(&self, idx: u16); + async fn resume(&self, idx: u16); async fn started_indices(&self) -> Vec; async fn stopped_indices(&self) -> Vec; async fn is_running(&self, idx: u16) -> bool; + /// Whether the executor at `idx` answers its gRPC health check right now. Stricter than + /// [`Self::is_running`]: a process that is aborting still counts as running until the OS has + /// finished with it, but it no longer serves. + async fn is_serving(&self, idx: u16) -> bool; async fn cluster_size(&self) -> u16; async fn stop_shard_manager(&self); @@ -954,6 +965,14 @@ impl WorkerExecutorClusterControl for EnvBasedTestDependencies { self.worker_executor_cluster.start(usize::from(idx)).await; } + async fn pause(&self, idx: u16) { + self.worker_executor_cluster.pause(usize::from(idx)).await; + } + + async fn resume(&self, idx: u16) { + self.worker_executor_cluster.resume(usize::from(idx)).await; + } + async fn started_indices(&self) -> Vec { self.worker_executor_cluster .started_indices() @@ -980,6 +999,19 @@ impl WorkerExecutorClusterControl for EnvBasedTestDependencies { worker_executor.is_running().await } + async fn is_serving(&self, idx: u16) -> bool { + let worker_executors = self.worker_executor_cluster.to_vec(); + let Some(worker_executor) = worker_executors.get(usize::from(idx)).cloned() else { + return false; + }; + crate::components::is_serving_grpc( + &worker_executor.grpc_host(), + worker_executor.grpc_port(), + Duration::from_secs(5), + ) + .await + } + async fn cluster_size(&self) -> u16 { Self::usize_to_u16(self.worker_executor_cluster.size()) } diff --git a/golem-test-framework/src/dsl/mod.rs b/golem-test-framework/src/dsl/mod.rs index d9946d55d9..981a2db9e1 100644 --- a/golem-test-framework/src/dsl/mod.rs +++ b/golem-test-framework/src/dsl/mod.rs @@ -1232,6 +1232,13 @@ pub fn worker_error_message(error: &WorkerExecutorError) -> String { match error { WorkerExecutorError::InvalidRequest { details } => details.clone(), WorkerExecutorError::PermissionDenied { details } => details.clone(), + WorkerExecutorError::OplogFenced { + agent_id, + expected_epoch, + actual_epoch, + } => format!( + "Oplog write for {agent_id:?} fenced: asserted epoch {expected_epoch}, stored {actual_epoch:?}" + ), WorkerExecutorError::AgentAlreadyExists { agent_id } => { format!("Worker already exists: {:?}", agent_id) } diff --git a/golem-worker-executor-test-utils/src/lib.rs b/golem-worker-executor-test-utils/src/lib.rs index f02b5efe85..1dffe56b87 100644 --- a/golem-worker-executor-test-utils/src/lib.rs +++ b/golem-worker-executor-test-utils/src/lib.rs @@ -683,7 +683,8 @@ impl TestWorkerExecutor { .ok_or_else(|| anyhow!("worker is not loaded: {owned_agent_id}"))?; golem_worker_executor::services::HasOplog::oplog(worker.as_ref()) .commit(CommitLevel::Always) - .await; + .await + .map_err(|error| anyhow!("oplog commit failed: {error}"))?; Ok(()) } @@ -719,8 +720,8 @@ impl TestWorkerExecutor { .await .ok_or_else(|| anyhow!("worker is not loaded: {owned_agent_id}"))?; let oplog = golem_worker_executor::services::HasOplog::oplog(worker.as_ref()); - let oplog_index = oplog.add(entry).await; - oplog.commit(CommitLevel::Always).await; + let oplog_index = oplog.add(entry).await?; + oplog.commit(CommitLevel::Always).await?; Ok(oplog_index) } @@ -755,7 +756,7 @@ impl TestWorkerExecutor { None, golem_common::base_model::oplog::QueuedCardEvent::revoke(card_id), )) - .await) + .await?) } pub async fn queue_card_install( @@ -3720,7 +3721,16 @@ impl TestOplog { #[async_trait] impl Oplog for TestOplog { - async fn add(&self, entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + entry: OplogEntry, + ) -> Result { + // Tests inject write failures by entry name; they used to go through `fallible_add`. + if let Err(details) = self.check_oplog_add(&entry).await { + return Err(golem_worker_executor::services::oplog::OplogError::Storage( + details, + )); + } if Self::is_consume_body_scope_start(&entry) && self.pause_before_consume_body_scope_start().await { @@ -3733,7 +3743,7 @@ impl Oplog for TestOplog { } let track_scope_start = Self::is_consume_body_scope_start(&entry); let gated = self.is_consume_body_chunk_data_end(&entry); - let index = self.oplog.add(entry.clone()).await; + let index = self.oplog.add(entry.clone()).await?; self.pause_after_fire_and_forget_rpc_checkpoint(index, &entry) .await; if track_scope_start @@ -3748,7 +3758,7 @@ impl Oplog for TestOplog { if gated { self.pause_at_consume_body_chunk_end_gate().await; } - index + Ok(index) } fn enqueue_add(&self, entry: OplogEntry) -> OplogAddReceipt { @@ -3756,41 +3766,31 @@ impl Oplog for TestOplog { let pending = self.oplog.enqueue_add(entry); let this = self.clone(); Box::pin(async move { - let index = pending.await; + let index = pending.await?; if gated { this.pause_at_consume_body_chunk_end_gate().await; } - index + Ok(index) }) } async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, golem_worker_executor::services::oplog::OplogError> + { self.oplog.add_durable_stream_batch(make_batch).await } - async fn fallible_add(&self, entry: OplogEntry) -> Result<(), String> { - self.check_oplog_add(&entry).await?; - self.oplog.fallible_add(entry).await - } - - async fn fallible_add_pair( - &self, - first: OplogEntry, - second: OplogEntry, - ) -> Result<(OplogIndex, OplogIndex), String> { - self.check_oplog_add(&first).await?; - self.check_oplog_add(&second).await?; - self.oplog.fallible_add_pair(first, second).await - } - async fn drop_prefix(&self, last_dropped_id: OplogIndex) -> u64 { self.oplog.drop_prefix(last_dropped_id).await } - async fn commit(&self, level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + level: CommitLevel, + ) -> Result, golem_worker_executor::services::oplog::OplogError> + { self.additional_test_deps .record_oplog_call(&self.owned_agent_id, "commit"); self.oplog.commit(level).await @@ -3919,7 +3919,7 @@ impl Oplog for TestOplog { &self, serialized_request: Vec, build_start: Box Result + Send>, - ) -> Result { + ) -> Result { let ordered = self .oplog .add_start_with_reserved_raw_payload(serialized_request, build_start) @@ -3949,7 +3949,7 @@ impl Oplog for TestOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let ordered = self .oplog .add_start_with_indexed_reserved_raw_payload(build_request) @@ -3980,7 +3980,14 @@ impl Oplog for TestOplog { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), golem_worker_executor::services::oplog::OplogError> { + // The second entry is only built once the first has its index, so the injected failure + // check covers the pair through the first entry alone. + if let Err(details) = self.check_oplog_add(&start).await { + return Err(golem_worker_executor::services::oplog::OplogError::Storage( + details, + )); + } self.oplog.add_pair(start, make_second).await } diff --git a/golem-worker-executor/benches/oplog_read.rs b/golem-worker-executor/benches/oplog_read.rs index 0ee3e40ef7..987c3df569 100644 --- a/golem-worker-executor/benches/oplog_read.rs +++ b/golem-worker-executor/benches/oplog_read.rs @@ -71,6 +71,7 @@ impl Fixture { self.initial_metadata.clone(), last_known_status(), execution_status(), + None, ) .await } @@ -172,13 +173,14 @@ async fn open_fixture(initial_entries: u64) -> Fixture { initial_metadata.clone(), last_known_status(), execution_status(), + None, ) .await; for value in 1..initial_entries { - oplog.add(entry(value)).await; + oplog.add(entry(value)).await.unwrap(); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); Fixture { oplog, @@ -196,7 +198,7 @@ async fn primary_fixture() -> Fixture { async fn buffered_fixture() -> Fixture { let fixture = open_fixture(ENTRY_COUNT - 8).await; for value in ENTRY_COUNT - 8..ENTRY_COUNT { - fixture.oplog.add(entry(value)).await; + fixture.oplog.add(entry(value)).await.unwrap(); } fixture } @@ -226,9 +228,9 @@ async fn cross_tier_fixture() -> Fixture { Some(true) ); for value in ARCHIVE_BOUNDARY..ENTRY_COUNT { - fixture.oplog.add(entry(value)).await; + fixture.oplog.add(entry(value)).await.unwrap(); } - fixture.oplog.commit(CommitLevel::Always).await; + fixture.oplog.commit(CommitLevel::Always).await.unwrap(); fixture } diff --git a/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql b/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql new file mode 100644 index 0000000000..54d48ad904 --- /dev/null +++ b/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql @@ -0,0 +1,13 @@ +-- The shard epoch authorised to write each oplog. +-- +-- One row per key in `index_storage`, carrying the ownership generation of the shard the agent +-- belongs to. An append asserts its epoch against this row inside the same transaction as the +-- insert, so an executor that has lost the shard cannot keep writing to an oplog whose owner has +-- moved on. A missing row fences too: the row is written before the first entry, and removed +-- before the entries are. +CREATE TABLE oplog_metadata ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + epoch BIGINT NOT NULL, + PRIMARY KEY (namespace, key) +); diff --git a/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql b/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql new file mode 100644 index 0000000000..8aa054dfb7 --- /dev/null +++ b/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql @@ -0,0 +1,7 @@ +-- The shard epoch authorised to write each oplog. See the postgres migration of the same name. +CREATE TABLE oplog_metadata ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + epoch INTEGER NOT NULL, + PRIMARY KEY (namespace, key) +); diff --git a/golem-worker-executor/src/durable_host/concurrent/call.rs b/golem-worker-executor/src/durable_host/concurrent/call.rs index 5d9146ffb9..62c67981f6 100644 --- a/golem-worker-executor/src/durable_host/concurrent/call.rs +++ b/golem-worker-executor/src/durable_host/concurrent/call.rs @@ -3230,7 +3230,7 @@ impl DurableCallSession { self.start_idx ))); } - oplog.add(end).await; + oplog.add(end).await?; self.execution_scope.release_atomic_lease(); DurableCallCoordinator::new(ctx) .finish(self.retry.function_type(), self.boundary, false) @@ -3439,13 +3439,13 @@ impl DurableCallSession { let end_append = oplog.enqueue_add(end); let post_end_append = post_end_entry.map(|entry| oplog.enqueue_add(entry)); let terminal = tokio::spawn(async move { - end_append.await; + end_append.await?; // A deferred-delivery call's mandatory post-`End` entry (e.g. its durable // `FinishSpan`) is appended by the same owned task: it is recorded even when the // completing future is torn right after the `End`, so replay can rely on it // unconditionally following the `End` (any discard marker chains after this task). if let Some(append) = post_end_append { - append.await; + append.await?; } Ok(()) }); @@ -4767,7 +4767,7 @@ where response: None, forced_commit: true, }) - .await; + .await?; } else if let Some(handle) = replay_handle { match replay_state.await_resolution_outcome(handle).await? { ResolutionOutcome::Resolved(Resolution::Completed { .. }) => {} @@ -4800,7 +4800,7 @@ where response: None, forced_commit: true, }) - .await; + .await?; } } } @@ -4916,7 +4916,7 @@ where if is_live { worker .add_to_oplog(OplogEntry::finish_span(parent_start_index, span_id.clone())) - .await; + .await?; } else { crate::get_oplog_entry_owned!(replay_state, OplogEntry::FinishSpan)?; } diff --git a/golem-worker-executor/src/durable_host/concurrent/delivery.rs b/golem-worker-executor/src/durable_host/concurrent/delivery.rs index aa1814353c..bbca8b524e 100644 --- a/golem-worker-executor/src/durable_host/concurrent/delivery.rs +++ b/golem-worker-executor/src/durable_host/concurrent/delivery.rs @@ -32,7 +32,7 @@ impl OrderedAppend { async fn wait(self) -> Result<(), WorkerExecutorError> { match self { Self::Receipt(receipt) => { - receipt.await; + receipt.await?; Ok(()) } Self::Task(task) => task.await.map_err(|err| { @@ -88,7 +88,16 @@ impl CompletionMarkerRecorder { let _ = done.send(Err(error)); return; } - let marker_idx = marker_append.await; + // Reported, not panicked: the process is built with `panic = "abort"`, so panicking + // in this task would take the whole executor down over one agent. The awaiter + // classifies a fenced marker as `ShardLost` and gives that agent up on its own. + let marker_idx = match marker_append.await { + Ok(index) => index, + Err(error) => { + let _ = done.send(Err(error.into())); + return; + } + }; match kind { CompletionMarkerKind::Delivered => { replay_state.record_delivered_completion(start_idx, marker_idx) diff --git a/golem-worker-executor/src/durable_host/concurrent/drop_events.rs b/golem-worker-executor/src/durable_host/concurrent/drop_events.rs index 5594f270b0..498199436c 100644 --- a/golem-worker-executor/src/durable_host/concurrent/drop_events.rs +++ b/golem-worker-executor/src/durable_host/concurrent/drop_events.rs @@ -116,7 +116,7 @@ impl DroppedCall { start_index: self.start_idx, partial, }; - oplog.add(cancelled).await; + oplog.add(cancelled).await?; Ok(()) } } diff --git a/golem-worker-executor/src/durable_host/concurrent/tests.rs b/golem-worker-executor/src/durable_host/concurrent/tests.rs index a3acbc06a2..cd18239698 100644 --- a/golem-worker-executor/src/durable_host/concurrent/tests.rs +++ b/golem-worker-executor/src/durable_host/concurrent/tests.rs @@ -363,7 +363,8 @@ async fn live_delivery_token( )))), durable_function_type: DurableFunctionType::ReadLocal, }) - .await; + .await + .unwrap(); seed_oplog .add(OplogEntry::End { timestamp: Timestamp::now_utc(), @@ -371,7 +372,8 @@ async fn live_delivery_token( response: None, forced_commit: false, }) - .await; + .await + .unwrap(); let seed_oplog_dyn: Arc = seed_oplog; let replay_state = ReplayState::new_for_owner( golem_common::model::OwnedAgentId { @@ -459,6 +461,59 @@ async fn completion_delivery_delivered_records_marker_via_drain() { } } +/// A completion marker whose oplog write is refused because the shard moved must be reported to +/// whoever awaits the receipt. Panicking here would abort the executor - and every other healthy +/// agent resident on it - over one agent that another executor now owns. +#[test] +async fn a_fenced_completion_marker_is_reported_rather_than_panicked() { + let agent_id = golem_common::model::AgentId { + component_id: golem_common::model::component::ComponentId::new(), + agent_id: "fenced-completion-marker-test".to_string(), + }; + let oplog = Arc::new(InMemoryOplog::fenced(crate::services::oplog::OplogFence { + agent_id: agent_id.clone(), + expected_epoch: golem_common::model::ShardEpoch(3), + actual_epoch: Some(golem_common::model::ShardEpoch(4)), + })); + let seed_oplog = Arc::new(InMemoryOplog::new()); + seed_oplog + .add(OplogEntry::NoOp { + timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, + }) + .await + .unwrap(); + let seed_oplog_dyn: Arc = seed_oplog; + let replay_state = ReplayState::new_for_owner( + golem_common::model::OwnedAgentId { + environment_id: golem_common::model::environment::EnvironmentId::new(), + agent_id, + }, + seed_oplog_dyn, + golem_common::model::regions::DeletedRegions::default(), + None, + crate::durable_host::tool::operation::OwnerToolOperations::new(), + ) + .await + .expect("failed to build replay state"); + let oplog_dyn: Arc = oplog; + let recorder = CompletionMarkerRecorder::new(oplog_dyn, replay_state); + + let mut receipt = recorder.record(idx(10), CompletionMarkerKind::Delivered, None); + + match await_marker_receipt(&mut receipt).await { + Err(WorkerExecutorError::OplogFenced { + expected_epoch, + actual_epoch, + .. + }) => { + assert_eq!(expected_epoch, 3); + assert_eq!(actual_epoch, Some(4)); + } + other => panic!("expected the fenced marker append to be reported, got {other:?}"), + } +} + #[test] async fn completion_delivery_markers_preserve_handoff_order() { let oplog = Arc::new(InMemoryOplog::new()); @@ -468,7 +523,8 @@ async fn completion_delivery_markers_preserve_handoff_order() { timestamp: Timestamp::now_utc(), entity_parent_start_index: None, }) - .await; + .await + .unwrap(); let seed_oplog_dyn: Arc = seed_oplog; let replay_state = ReplayState::new_for_owner( golem_common::model::OwnedAgentId { @@ -682,7 +738,8 @@ async fn tail_gated_token_over_crash_tail( timestamp: Timestamp::now_utc(), entity_parent_start_index: None, }) - .await; + .await + .unwrap(); oplog .add(OplogEntry::Start { timestamp: Timestamp::now_utc(), @@ -695,7 +752,8 @@ async fn tail_gated_token_over_crash_tail( )))), durable_function_type: DurableFunctionType::ReadLocal, }) - .await; + .await + .unwrap(); oplog .add(OplogEntry::End { timestamp: Timestamp::now_utc(), @@ -707,9 +765,10 @@ async fn tail_gated_token_over_crash_tail( ))), forced_commit: false, }) - .await; + .await + .unwrap(); for entry in extra_tail { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog_dyn: Arc = oplog.clone(); let replay_state = ReplayState::new_for_owner( @@ -951,6 +1010,9 @@ struct InMemoryOplog { next_reserved: Arc, next_commit: Arc>, append_progress: Arc, + /// When set, every append is refused the way a fencing backend refuses one whose asserted + /// shard epoch is stale: the shard moved to another executor while this oplog was open. + fence: Option, } impl InMemoryOplog { @@ -961,6 +1023,15 @@ impl InMemoryOplog { next_reserved: Arc::new(std::sync::atomic::AtomicU64::new(1)), next_commit: Arc::new(tokio::sync::Mutex::new(1)), append_progress: Arc::new(tokio::sync::Notify::new()), + fence: None, + } + } + + /// An oplog whose shard has already moved: every append is refused with `fence`. + fn fenced(fence: crate::services::oplog::OplogFence) -> Self { + Self { + fence: Some(fence), + ..Self::new() } } @@ -974,17 +1045,24 @@ impl InMemoryOplog { next_reserved: Arc::new(std::sync::atomic::AtomicU64::new(1)), next_commit: Arc::new(tokio::sync::Mutex::new(1)), append_progress: Arc::new(tokio::sync::Notify::new()), + fence: None, } } } #[async_trait] impl Oplog for InMemoryOplog { - async fn add(&self, entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + entry: OplogEntry, + ) -> Result { self.enqueue_add(entry).await } fn enqueue_add(&self, entry: OplogEntry) -> crate::services::oplog::OplogAddReceipt { + if let Some(fence) = self.fence.clone() { + return Box::pin(async move { Err(crate::services::oplog::OplogError::Fenced(fence)) }); + } let index = self .next_reserved .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -1025,9 +1103,11 @@ impl Oplog for InMemoryOplog { } }); Box::pin(async move { - receipt - .await - .expect("the in-memory oplog append task must reply") + Ok({ + receipt + .await + .expect("the in-memory oplog append task must reply") + }) }) } @@ -1035,7 +1115,7 @@ impl Oplog for InMemoryOplog { &self, _start: OplogEntry, _make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { // The concurrent (p3) durability path under test never writes sequential-adapter pairs, // and a routed-through-`add` implementation could not honor the atomic pair contract. unreachable!("add_pair is not used by the concurrent durability tests") @@ -1048,11 +1128,11 @@ impl Oplog for InMemoryOplog { dyn FnOnce(golem_common::model::oplog::RawOplogPayload) -> Result + Send, >, - ) -> Result { + ) -> Result { let entry = build_start( golem_common::model::oplog::RawOplogPayload::SerializedInline(serialized_request), )?; - let index = self.add(entry.clone()).await; + let index = self.add(entry.clone()).await?; Ok(crate::services::oplog::OrderedOplogStart { index, entry, @@ -1063,7 +1143,7 @@ impl Oplog for InMemoryOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: crate::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let mut entries = self.entries.lock().await; let index = OplogIndex::from_u64(entries.len() as u64 + 1); let (serialized_request, build_start) = build_request(index)?; @@ -1085,8 +1165,11 @@ impl Oplog for InMemoryOplog { async fn commit( &self, _level: CommitLevel, - ) -> std::collections::BTreeMap { - std::collections::BTreeMap::new() + ) -> Result< + std::collections::BTreeMap, + crate::services::oplog::OplogError, + > { + Ok(std::collections::BTreeMap::new()) } async fn current_oplog_index(&self) -> OplogIndex { @@ -1162,7 +1245,8 @@ async fn dropped_cancellable_call_records_cancelled_at_next_drain_point() { )))), durable_function_type: DurableFunctionType::ReadLocal, }) - .await; + .await + .unwrap(); let (tx, mut rx) = mpsc::unbounded_channel(); { @@ -1224,7 +1308,8 @@ async fn access_terminal_end_is_appended_before_cleanup_and_permit_release() { )))), durable_function_type: DurableFunctionType::ReadRemote, }) - .await; + .await + .unwrap(); let permit_counter = Arc::new(AtomicUsize::new(0)); let (cleanup_tx, mut cleanup_rx) = mpsc::unbounded_channel(); diff --git a/golem-worker-executor/src/durable_host/durable_session.rs b/golem-worker-executor/src/durable_host/durable_session.rs index 53e7fca21a..c9fab18c32 100644 --- a/golem-worker-executor/src/durable_host/durable_session.rs +++ b/golem-worker-executor/src/durable_host/durable_session.rs @@ -5134,7 +5134,7 @@ mod tests { #[async_trait::async_trait] impl DurableStreamConsumerJournal for TestConsumerJournal { async fn commit(&self) -> Result<(), String> { - self.0.commit(CommitLevel::Always).await; + self.0.commit(CommitLevel::Always).await.unwrap(); Ok(()) } } @@ -5197,6 +5197,7 @@ mod tests { Vec::new(), )) .await + .expect("oplog write") } #[test] @@ -6024,7 +6025,7 @@ mod tests { #[async_trait::async_trait] impl DurableStreamConsumerJournal for RecordingConsumerJournal { async fn commit(&self) -> Result<(), String> { - self.oplog.commit(CommitLevel::Always).await; + self.oplog.commit(CommitLevel::Always).await.unwrap(); self.commits.fetch_add(1, Ordering::Relaxed); Ok(()) } @@ -6991,7 +6992,10 @@ mod tests { assert_eq!(intents[0].reason, StreamCancelReasonV1::GuestDrop); for _ in 0..2050 { - oplog.add(OplogEntry::interrupted()).await; + oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } drop(guest.current_control_metadata().await.unwrap()); oplog.take_read_ranges(); @@ -7472,7 +7476,8 @@ mod tests { Vec::new(), Vec::new(), )) - .await; + .await + .unwrap(); producer .append_session_record(StreamSessionRecordV1::Attached( StreamSessionAttachedRecordV1 { @@ -7979,7 +7984,8 @@ mod tests { Vec::new(), Vec::new(), )) - .await; + .await + .unwrap(); streams .append_record(StreamSessionRecordV1::Attached( golem_common::base_model::durable_stream::StreamSessionAttachedRecordV1 { @@ -8084,7 +8090,10 @@ mod tests { assert_eq!(producer_oplog.current_oplog_index().await, producer_length); for _ in 0..2050 { - consumer_oplog.add(OplogEntry::interrupted()).await; + consumer_oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } assert_eq!( streams @@ -8458,7 +8467,8 @@ mod tests { Vec::new(), Vec::new(), )) - .await; + .await + .unwrap(); producer .append_session_record(StreamSessionRecordV1::Attached( StreamSessionAttachedRecordV1 { @@ -8800,7 +8810,8 @@ mod tests { Vec::new(), Vec::new(), )) - .await; + .await + .unwrap(); let streams = DurableSessionStreams::new( producer.clone(), oplog.clone(), @@ -9184,7 +9195,8 @@ mod tests { Vec::new(), Vec::new(), )) - .await; + .await + .unwrap(); streams .append_record(StreamSessionRecordV1::Attached( StreamSessionAttachedRecordV1 { @@ -9314,7 +9326,10 @@ mod tests { let streams = DurableSessionStreams::new(producer, oplog.clone(), identity.invocation.clone(), []); for _ in 0..2050 { - oplog.add(OplogEntry::interrupted()).await; + oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } assert!( streams @@ -9348,11 +9363,15 @@ mod tests { }, ))), }) - .await; + .await + .expect("oplog write"); // No commit: another local append must already be visible. assert_eq!(streams.caller_attempt_id().await.unwrap(), attempt_id); assert_eq!(oplog.take_read_ranges(), vec![(index, 1)]); - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!( streams.clone().caller_attempt_id().await.unwrap(), attempt_id @@ -9375,7 +9394,10 @@ mod tests { .unwrap(); let streams = DurableSessionStreams::new(producer, oplog.clone(), identity.invocation, []); for _ in 0..2050 { - oplog.add(OplogEntry::interrupted()).await; + oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } streams.recover_session_mappings().await.unwrap(); assert_eq!( @@ -9384,7 +9406,10 @@ mod tests { ); streams.clone().recover_session_mappings().await.unwrap(); assert!(oplog.take_read_ranges().is_empty()); - let next = oplog.add(OplogEntry::interrupted()).await; + let next = oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); streams.recover_session_mappings().await.unwrap(); assert_eq!(oplog.take_read_ranges(), vec![(next, 1)]); } @@ -9491,7 +9516,8 @@ mod tests { }, ))), }) - .await; + .await + .expect("oplog write"); assert_eq!( streams.persisted_finished().await.unwrap(), diff --git a/golem-worker-executor/src/durable_host/durable_stream.rs b/golem-worker-executor/src/durable_host/durable_stream.rs index 767942a0fa..8dea7d8c8d 100644 --- a/golem-worker-executor/src/durable_host/durable_stream.rs +++ b/golem-worker-executor/src/durable_host/durable_stream.rs @@ -1832,7 +1832,10 @@ impl DurableStreamProducer { let commit: DurableStreamCommit = Arc::new(move |committed| { let oplog = commit_oplog.clone(); Box::pin(async move { - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); if let Some(committed) = committed { let _ = committed.send(()); } @@ -1899,10 +1902,9 @@ impl DurableStreamProducer { record, .. } => { - let record = oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; if matches!( &record.coordinate, StreamRegistrationCoordinateV1::Nested { .. } @@ -1934,10 +1936,9 @@ impl DurableStreamProducer { record, .. } => { - let record = oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; index.apply_item_batch( oplog_index, entity_parent_start_index, @@ -1959,10 +1960,9 @@ impl DurableStreamProducer { .to_string(), )); } - let record = oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; index.apply_end( oplog_index, entity_parent_start_index, @@ -1981,10 +1981,9 @@ impl DurableStreamProducer { .to_string(), )); } - let record = oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; index.apply_cancel( oplog_index, entity_parent_start_index, @@ -2003,10 +2002,9 @@ impl DurableStreamProducer { .to_string(), )); } - let record = oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; index.apply_session_references(entity_parent_start_index, &record)?; index.apply_result_offset(oplog_index, &record); index.apply_deletion_record( @@ -2402,7 +2400,8 @@ impl DurableStreamProducer { entity_parent_start_index, OplogPayload::Inline(Box::new(record)), )) - .await; + .await + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; if let Some(key) = result_key { index.invocation_results.entry(key).or_insert(oplog_index); } @@ -2446,7 +2445,7 @@ impl DurableStreamProducer { .oplog .download_payload(record) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; match record { StreamSessionRecordV1::ConsumerItemValue(record) if record.session_key == key.session_key @@ -2532,7 +2531,8 @@ impl DurableStreamProducer { entity_parent_start_index, OplogPayload::Inline(Box::new(record)), )) - .await; + .await + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; self.notify_session_records_changed(); Ok(false) @@ -2598,7 +2598,8 @@ impl DurableStreamProducer { entity_parent_start_index, OplogPayload::Inline(Box::new(record)), )) - .await; + .await + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; *index = updated; } @@ -2758,7 +2759,7 @@ impl DurableStreamProducer { )] })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let (oplog_index, entry) = entries .pop() @@ -2770,7 +2771,7 @@ impl DurableStreamProducer { .oplog .download_payload(record) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; index.apply_registration( oplog_index, entity_parent_start_index, @@ -2911,7 +2912,7 @@ impl DurableStreamProducer { result })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; let mut prepared = None; let mut registrations = Vec::with_capacity(requests.len()); @@ -2922,19 +2923,17 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; registrations.push((oplog_index, entity_parent_start_index, record)); } OplogEntry::StreamSession { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; match record { StreamSessionRecordV1::Prepared(record) => prepared = Some(record), StreamSessionRecordV1::Attached(_) => {} @@ -3111,7 +3110,7 @@ impl DurableStreamProducer { )] })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let (oplog_index, entry) = entries.pop().ok_or_else(|| { DurableStreamProducerError::CorruptHistory( @@ -3241,7 +3240,7 @@ impl DurableStreamProducer { result })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let mut handles = Vec::new(); @@ -3253,11 +3252,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; handles.push(record.handle.clone()); index.apply_registration( oplog_index, @@ -3280,11 +3278,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; index.apply_session_references(entity_parent_start_index, &record)?; index.apply_result_offset(oplog_index, &record); session_record = Some(record); @@ -3940,7 +3937,7 @@ impl DurableStreamProducer { records })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let mut pending_registrations = Vec::new(); @@ -3952,11 +3949,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; pending_registrations.push((oplog_index, entity_parent_start_index, record)); } OplogEntry::StreamItems { @@ -3964,11 +3960,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; committed_item = Some((oplog_index, entity_parent_start_index, record)); } OplogEntry::StreamSession { .. } => {} @@ -4053,7 +4048,7 @@ impl DurableStreamProducer { )] })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let (oplog_index, entry) = entries .pop() @@ -4070,7 +4065,7 @@ impl DurableStreamProducer { .oplog .download_payload(record) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; let event = index.apply_end( oplog_index, entity_parent_start_index, @@ -4192,7 +4187,7 @@ impl DurableStreamProducer { )] })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let (oplog_index, entry) = entries .pop() @@ -4209,7 +4204,7 @@ impl DurableStreamProducer { .oplog .download_payload(record) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; let event = index.apply_end( oplog_index, entity_parent_start_index, @@ -4326,7 +4321,7 @@ impl DurableStreamProducer { )] })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let (oplog_index, entry) = entries .pop() @@ -4343,7 +4338,7 @@ impl DurableStreamProducer { .oplog .download_payload(record) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; let event = index.apply_cancel( oplog_index, entity_parent_start_index, @@ -4660,7 +4655,7 @@ impl DurableStreamProducer { records })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let mut terminal_events = Vec::new(); @@ -4671,11 +4666,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; terminal_events.push(index.apply_end( oplog_index, entity_parent_start_index, @@ -4688,11 +4682,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; terminal_events.push(index.apply_cancel( oplog_index, entity_parent_start_index, @@ -4701,11 +4694,10 @@ impl DurableStreamProducer { )?); } OplogEntry::StreamSession { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; let StreamSessionRecordV1::Finished(record) = record else { return Err(DurableStreamProducerError::CorruptHistory( "session finish batch contains an unexpected session record" @@ -5632,7 +5624,7 @@ impl StreamAttachmentConsumerProbe for DbDirectStreamAttachmentConsumerProbe { .oplog_service .download_payload(&consumer, AgentMode::Durable, record) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; match record { StreamSessionRecordV1::ConsumerItemValue(record) if record.session_key == key.session_key @@ -6120,7 +6112,7 @@ impl DurableStreamProducer { records })) .await - .map_err(DurableStreamProducerError::Oplog)?; + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; let mut terminal_events = Vec::new(); for (oplog_index, entry) in entries { @@ -6130,11 +6122,10 @@ impl DurableStreamProducer { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; terminal_events.push(index.apply_cancel( oplog_index, entity_parent_start_index, @@ -6143,11 +6134,10 @@ impl DurableStreamProducer { )?); } OplogEntry::StreamSession { record, .. } => { - let record = self - .oplog - .download_payload(record) - .await - .map_err(DurableStreamProducerError::Oplog)?; + let record = + self.oplog.download_payload(record).await.map_err(|error| { + DurableStreamProducerError::Oplog(error.to_string()) + })?; index.apply_deletion_record( &record, self.environment_id, @@ -6228,7 +6218,8 @@ impl DurableStreamProducer { entity_parent_start_index, OplogPayload::Inline(Box::new(record.clone())), )) - .await; + .await + .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; self.commit().await; index.apply_session_references(entity_parent_start_index, &record)?; index.apply_deletion_record( @@ -7013,14 +7004,17 @@ pub(crate) mod tests { #[async_trait] impl Oplog for TestOplog { - async fn add(&self, entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + entry: OplogEntry, + ) -> Result { let mut state = self.state.lock().unwrap(); let index = state .entries .last_key_value() .map_or(OplogIndex::INITIAL, |(index, _)| index.next()); state.entries.insert(index, entry); - index + Ok(index) } fn enqueue_add(&self, entry: OplogEntry) -> OplogAddReceipt { @@ -7030,7 +7024,7 @@ pub(crate) mod tests { .last_key_value() .map_or(OplogIndex::INITIAL, |(index, _)| index.next()); state.entries.insert(index, entry); - Box::pin(async move { index }) + Box::pin(async move { Ok(index) }) } async fn drop_prefix(&self, last_dropped_id: OplogIndex) -> u64 { @@ -7040,20 +7034,23 @@ pub(crate) mod tests { (before - state.entries.len()) as u64 } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, crate::services::oplog::OplogError> { let mut state = self.state.lock().unwrap(); let committed = state .entries .iter() .filter(|(index, _)| **index > state.committed) .map(|(index, entry)| (*index, entry.clone())) - .collect(); + .collect::>(); state.committed = state .entries .last_key_value() .map_or(state.committed, |(index, _)| *index); state.commit_count += 1; - committed + Ok(committed) } async fn current_oplog_index(&self) -> OplogIndex { @@ -7175,9 +7172,9 @@ pub(crate) mod tests { &self, serialized_request: Vec, build_start: Box Result + Send>, - ) -> Result { + ) -> Result { let entry = build_start(RawOplogPayload::SerializedInline(serialized_request))?; - let index = self.add(entry.clone()).await; + let index = self.add(entry.clone()).await?; Ok(OrderedOplogStart { index, entry, @@ -7188,7 +7185,7 @@ pub(crate) mod tests { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: crate::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let mut state = self.state.lock().unwrap(); let index = state .entries @@ -7208,10 +7205,10 @@ pub(crate) mod tests { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { - let first = self.add(start).await; - let second = self.add(make_second(first)).await; - (first, second) + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { + let first = self.add(start).await?; + let second = self.add(make_second(first)).await?; + Ok((first, second)) } } @@ -7219,7 +7216,7 @@ pub(crate) mod tests { async fn test_oplog_read_exact_includes_uncommitted_entries() { let oplog = TestOplog::default(); let entry = OplogEntry::interrupted(); - let index = oplog.add(entry.clone()).await; + let index = oplog.add(entry.clone()).await.unwrap(); let entries = oplog.read_exact(index, 1).await; @@ -7229,7 +7226,7 @@ pub(crate) mod tests { #[test] async fn test_oplog_read_exact_rejects_incomplete_range() { let oplog = TestOplog::default(); - let index = oplog.add(OplogEntry::interrupted()).await; + let index = oplog.add(OplogEntry::interrupted()).await.unwrap(); let result = std::panic::AssertUnwindSafe(oplog.read_exact(index, 2)) .catch_unwind() @@ -7722,7 +7719,10 @@ pub(crate) mod tests { request.entity_parent_start_index = entity_parent_start_index; let handle = live.register(request).await.unwrap().value; - oplog.add(OplogEntry::no_op(None)).await; + oplog + .add(OplogEntry::no_op(None)) + .await + .expect("oplog write"); live.write_items(handle.stream_id, 0, StreamItemsPayloadV1::PackedU8(vec![1])) .await .unwrap(); @@ -7779,7 +7779,10 @@ pub(crate) mod tests { .unwrap(); } for _ in 0..2050 { - oplog.add(OplogEntry::interrupted()).await; + oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } assert!( producer.index.lock().await.streams[&handle.stream_id] @@ -8004,7 +8007,10 @@ pub(crate) mod tests { .unwrap() .value; for _ in 0..2050 { - oplog.add(OplogEntry::interrupted()).await; + oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } producer .end(handle.stream_id, 3, StreamEndResultV1::Ok) @@ -9173,7 +9179,7 @@ pub(crate) mod tests { let oplog = oplog_for_commit.clone(); let batches = batches_for_commit.clone(); Box::pin(async move { - let committed_entries = oplog.commit(CommitLevel::Always).await; + let committed_entries = oplog.commit(CommitLevel::Always).await.unwrap(); batches .lock() .unwrap() @@ -9209,7 +9215,8 @@ pub(crate) mod tests { Vec::new(), Vec::new(), )) - .await; + .await + .unwrap(); producer .append_session_record(StreamSessionRecordV1::ConsumerTerminal( golem_common::model::durable_stream::StreamConsumerTerminalRecordV1 { @@ -9418,7 +9425,7 @@ pub(crate) mod tests { let oplog = oplog.clone(); let commit_reached = commit_reached.clone(); Box::pin(async move { - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); if let Some(committed) = committed { let _ = committed.send(()); } @@ -9932,7 +9939,7 @@ pub(crate) mod tests { })) .await .unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); drop(producer); assert!(matches!( @@ -10078,7 +10085,7 @@ pub(crate) mod tests { })) .await .unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); drop(producer); assert!( @@ -10131,7 +10138,7 @@ pub(crate) mod tests { })) .await .unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); drop(producer); assert!(matches!( @@ -10528,7 +10535,7 @@ pub(crate) mod tests { let oplog = oplog.clone(); let commit_reached = commit_reached.clone(); Box::pin(async move { - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); if let Some(committed) = committed { let _ = committed.send(()); } diff --git a/golem-worker-executor/src/durable_host/durable_stream/metadata.rs b/golem-worker-executor/src/durable_host/durable_stream/metadata.rs index 460ebfbf9a..51bc304b52 100644 --- a/golem-worker-executor/src/durable_host/durable_stream/metadata.rs +++ b/golem-worker-executor/src/durable_host/durable_stream/metadata.rs @@ -1539,6 +1539,7 @@ mod tests { timestamp: Timestamp::now_utc(), }, ))), + None, ) .await; let service = Arc::new(DefaultWorkerService::new( @@ -1562,7 +1563,10 @@ mod tests { let commit: DurableStreamCommit = Arc::new(move |published| { let oplog = oplog.clone(); Box::pin(async move { - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); if let Some(published) = published { let _ = published.send(()); } @@ -1599,7 +1603,10 @@ mod tests { async fn persist(&self) { let owner = OwnedAgentId::new(self.identity.environment_id, &self.identity.agent_id); - self.oplog.commit(CommitLevel::Always).await; + self.oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); self.service .lookup_durable_stream_producer_metadata( &owner, @@ -1805,7 +1812,11 @@ mod tests { offsets.push(outcome.value[0]); } for _ in 0..2100 { - fixture.oplog.add(OplogEntry::interrupted()).await; + fixture + .oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } fixture.persist().await; drop(producer); @@ -2135,7 +2146,11 @@ mod tests { .unwrap() .value; for _ in 0..1021 { - fixture.oplog.add(OplogEntry::interrupted()).await; + fixture + .oplog + .add(OplogEntry::interrupted()) + .await + .expect("oplog write"); } assert_eq!(fixture.oplog.current_oplog_index().await.as_u64(), 1023); let nested = registration( @@ -2159,7 +2174,11 @@ mod tests { .await .unwrap(); let nested_handles = producer.nested_handles(handle.stream_id, 0).await.unwrap(); - fixture.oplog.commit(CommitLevel::Always).await; + fixture + .oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); MultiLayerOplog::try_archive_blocking(&fixture.oplog) .await .expect("archive layer"); diff --git a/golem-worker-executor/src/durable_host/golem/v1x.rs b/golem-worker-executor/src/durable_host/golem/v1x.rs index 8c52d532c1..f9220639f0 100644 --- a/golem-worker-executor/src/durable_host/golem/v1x.rs +++ b/golem-worker-executor/src/durable_host/golem/v1x.rs @@ -626,6 +626,7 @@ impl Host for DurableWorkerCtx { .oplog .add(OplogEntry::no_op(self.entity_parent_start_index())) .await + .map_err(|error| anyhow!(WorkerExecutorError::from(error)))? { OplogIndex::NONE => self.state.current_oplog_index().await, index => index, @@ -766,6 +767,7 @@ impl Host for DurableWorkerCtx { self.entity_parent_start_index(), )) .await + .map_err(|error| anyhow!(WorkerExecutorError::from(error)))? { OplogIndex::NONE => self.state.current_oplog_index().await, index => index, @@ -889,7 +891,7 @@ impl Host for DurableWorkerCtx { self.entity_parent_start_index(), begin_index, )) - .await; + .await?; } else { let (_, _) = get_oplog_entry!(self.state.replay_state, OplogEntry::EndAtomicRegion)?; } diff --git a/golem-worker-executor/src/durable_host/logging/policy.rs b/golem-worker-executor/src/durable_host/logging/policy.rs index b02ee9872f..4dde7c94dc 100644 --- a/golem-worker-executor/src/durable_host/logging/policy.rs +++ b/golem-worker-executor/src/durable_host/logging/policy.rs @@ -112,7 +112,10 @@ pub async fn emit_log_event_with_state( if !replay_state.seen_log(*level, context, message).await { // haven't seen this log before public_state.event_service().emit_event(event.clone(), true); - public_state.worker().add_to_oplog(entry).await; + public_state + .worker() + .add_to_oplog_or_relinquish(entry) + .await; } else { // we have persisted emitting this log before, so we mark it as non-live and // remove the entry from the seen log set. @@ -129,7 +132,18 @@ pub async fn emit_log_event_with_state( public_state.event_service().emit_event(event.clone(), true); if is_live && !replay_state.seen_log(*level, context, message).await { - oplog.add(entry).await; + // Same contract as `Worker::add_to_oplog_or_relinquish`, spelled out + // because this writes through the oplog handle passed in rather than the + // worker's own: a fence gives the agent up, anything else is fail-stop. + match oplog.add(entry).await { + Ok(_) => {} + Err(crate::services::oplog::OplogError::Fenced(fence)) => { + public_state.worker().mark_relinquished( + crate::worker::RelinquishReason::Fenced(Some(Box::new(fence))), + ); + } + Err(error) => panic!("oplog write: {error}"), + } } } } diff --git a/golem-worker-executor/src/durable_host/mod.rs b/golem-worker-executor/src/durable_host/mod.rs index 1d97840a9c..e2d7d7e901 100644 --- a/golem-worker-executor/src/durable_host/mod.rs +++ b/golem-worker-executor/src/durable_host/mod.rs @@ -2149,7 +2149,7 @@ impl DurableWorkerCtx { if commit_immediately { self.public_state.worker().add_and_commit_oplog(entry).await; } else { - self.public_state.worker().add_to_oplog(entry).await; + self.public_state.worker().add_to_oplog(entry).await?; } Ok(()) @@ -2467,6 +2467,9 @@ impl DurableWorkerCtx { TrapType::Interrupt(InterruptKind::Suspend(ts)) => Some(RetryDecision::TryStop(*ts)), TrapType::Interrupt(InterruptKind::Restart) => Some(RetryDecision::Immediate), TrapType::Interrupt(InterruptKind::Jump) => Some(RetryDecision::Immediate), + // Never retried here: a retry in place would reopen the oplog with the same stale + // epoch. The worker service resumes the agent on the shard's owner. + TrapType::Interrupt(InterruptKind::ShardLost) => Some(RetryDecision::None), TrapType::Exit => Some(RetryDecision::None), TrapType::Error { error: AgentError::OutOfMemory, @@ -2956,7 +2959,7 @@ impl DurableWorkerCtx { response: None, forced_commit: true, }; - self.state.oplog.add(entry).await; + self.state.oplog.add(entry).await?; // The durable scope opened in `begin_function` is now closed. self.state.remove_durable_scope(begin_index)?; } else { @@ -3034,7 +3037,7 @@ impl DurableWorkerCtx { response: None, forced_commit: true, }) - .await; + .await?; } } } @@ -3096,7 +3099,8 @@ impl DurableWorkerCtx { scope_start, Box::new(move |_start_index| OplogEntry::begin_remote_transaction(tx_id, None)), ) - .await; + .await + .map_err(WorkerExecutorError::from)?; self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) @@ -3318,9 +3322,8 @@ impl DurableWorkerCtx { // make sure to write to the local oplog handle, but still commit to the parent for status consistency. self.state .oplog - .fallible_add(OplogEntry::pre_commit_remote_transaction(begin_index)) - .await - .map_err(WorkerExecutorError::runtime)?; + .add(OplogEntry::pre_commit_remote_transaction(begin_index)) + .await?; self.public_state .worker() @@ -3347,9 +3350,8 @@ impl DurableWorkerCtx { // make sure to write to the local oplog handle, but still commit to the parent for status consistency. self.state .oplog - .fallible_add(OplogEntry::pre_rollback_remote_transaction(begin_index)) - .await - .map_err(WorkerExecutorError::runtime)?; + .add(OplogEntry::pre_rollback_remote_transaction(begin_index)) + .await?; self.public_state .worker() @@ -3470,17 +3472,16 @@ impl DurableWorkerCtx { // successful append can never leave one without the other. self.state .oplog - .fallible_add_pair( + .add_pair( marker, - OplogEntry::End { + Box::new(move |_| OplogEntry::End { timestamp: Timestamp::now_utc(), start_index: begin_index, response: None, forced_commit: true, - }, + }), ) - .await - .map_err(WorkerExecutorError::runtime)?; + .await?; self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) @@ -4807,6 +4808,16 @@ impl InvocationHooks for DurableWorkerCtx { ) -> RetryDecision { let current_idempotency_key = self.get_current_idempotency_key().await; + // Deliberately above the dropped-call drain: that drain appends `Cancelled` entries, and + // a relinquished agent's oplog belongs to another executor now. Nothing further is + // written for it - not the drain, not an `Error` entry, not a status change. + if matches!(trap_type, TrapType::Interrupt(InterruptKind::ShardLost)) { + self.public_state + .worker() + .mark_relinquished(crate::worker::RelinquishReason::Fenced(None)); + return RetryDecision::None; + } + if self.state.is_live() && !self.state.snapshotting_mode && let Err(err) = concurrent::drain_queued_dropped_call_events(self).await @@ -4859,7 +4870,8 @@ impl InvocationHooks for DurableWorkerCtx { }, ) = (¤t_idempotency_key, trap_type) { - self.state + let denial_persisted = self + .state .oplog .add_pair( OplogEntry::cancel_pending_invocation(idempotency_key.clone()), @@ -4880,6 +4892,19 @@ impl InvocationHooks for DurableWorkerCtx { }), ) .await; + match denial_persisted { + Ok(_) => {} + Err(crate::services::oplog::OplogError::Fenced(fence)) => { + // The shard moved while this failure was being recorded. Give the agent up + // exactly as the `ShardLost` arm above does: nothing further may be written + // to an oplog that belongs to another executor now. + self.public_state.worker().mark_relinquished( + crate::worker::RelinquishReason::Fenced(Some(Box::new(fence))), + ); + return RetryDecision::None; + } + Err(error) => panic!("oplog write: {error}"), + } self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) @@ -4894,6 +4919,8 @@ impl InvocationHooks for DurableWorkerCtx { TrapType::Interrupt(InterruptKind::Suspend(_)) => Some(OplogEntry::suspend()), TrapType::Interrupt(InterruptKind::Jump) => None, TrapType::Interrupt(InterruptKind::Restart) => None, + // The oplog is the new owner's; a stale writer must leave no trace in it. + TrapType::Interrupt(InterruptKind::ShardLost) => None, TrapType::Exit => Some(OplogEntry::exited()), TrapType::Error { error: AgentError::PermissionDenied(_), @@ -5171,7 +5198,10 @@ impl ResourceStore for DurableWorkerCtx { resource_id, name.clone(), ); - self.public_state.worker().add_to_oplog(entry).await; + self.public_state + .worker() + .add_to_oplog_or_relinquish(entry) + .await; } id } @@ -5186,7 +5216,10 @@ impl ResourceStore for DurableWorkerCtx { id, resource_type_id.clone(), ); - self.public_state.worker().add_to_oplog(entry).await; + self.public_state + .worker() + .add_to_oplog_or_relinquish(entry) + .await; } } result @@ -5351,7 +5384,7 @@ impl InvocationContextManagement for DurableWorkerCtx { linked_context_id: span.linked_context().map(|link| link.span_id().clone()), attributes: HashMap::from_iter(initial_attributes.iter().cloned()).into(), }) - .await; + .await?; } Ok(span) @@ -5388,7 +5421,7 @@ impl InvocationContextManagement for DurableWorkerCtx { self.entity_parent_start_index(), span_id.clone(), )) - .await; + .await?; } else if !self.is_live() { crate::get_oplog_entry!(self.state.replay_state, OplogEntry::FinishSpan)?; } @@ -5431,7 +5464,7 @@ impl InvocationContextManagement for DurableWorkerCtx { key.to_string(), value, )) - .await; + .await?; } else if !self.is_live() { crate::get_oplog_entry!(self.state.replay_state, OplogEntry::SetSpanAttribute)?; } @@ -5944,7 +5977,7 @@ impl ExternalOperations for DurableWorkerCtx { .get_public_state() .oplog() .add(OplogEntry::restart()) - .await; + .await?; Ok(None) } else { diff --git a/golem-worker-executor/src/durable_host/p3/http/request_body.rs b/golem-worker-executor/src/durable_host/p3/http/request_body.rs index b4c1672670..59a16fa7aa 100644 --- a/golem-worker-executor/src/durable_host/p3/http/request_body.rs +++ b/golem-worker-executor/src/durable_host/p3/http/request_body.rs @@ -464,13 +464,14 @@ pub(super) async fn record_frame_entry( let bytes = serialize(&request)?; let raw = oplog.upload_raw_payload(bytes).await?; let payload = raw.into_payload::()?; - Ok(oplog + oplog .add(OplogEntry::host_stream_frame( parent_start_index, HostStreamKind::P3HttpRequestBody, payload, )) - .await) + .await + .map_err(|error| error.to_string()) } /// Loads one recorded data/trailers frame back from its `HostStreamFrame` diff --git a/golem-worker-executor/src/durable_host/p3/http/response_body.rs b/golem-worker-executor/src/durable_host/p3/http/response_body.rs index 371379005f..7a38eac5ab 100644 --- a/golem-worker-executor/src/durable_host/p3/http/response_body.rs +++ b/golem-worker-executor/src/durable_host/p3/http/response_body.rs @@ -2001,7 +2001,8 @@ mod tests { )))), durable_function_type: DurableFunctionType::WriteRemoteBatched(None), }) - .await; + .await + .unwrap(); let child_start = oplog .add(OplogEntry::Start { timestamp: Timestamp::now_utc(), @@ -2016,7 +2017,8 @@ mod tests { OplogIndex::from_u64(1), )), }) - .await; + .await + .unwrap(); oplog .add(OplogEntry::End { timestamp: Timestamp::now_utc(), @@ -2030,7 +2032,8 @@ mod tests { ))), forced_commit: false, }) - .await; + .await + .unwrap(); child_start } diff --git a/golem-worker-executor/src/durable_host/p3/http/test_support.rs b/golem-worker-executor/src/durable_host/p3/http/test_support.rs index 39b25fd6c1..499d02515b 100644 --- a/golem-worker-executor/src/durable_host/p3/http/test_support.rs +++ b/golem-worker-executor/src/durable_host/p3/http/test_support.rs @@ -119,44 +119,47 @@ impl FrameTestOplog { #[async_trait] impl Oplog for FrameTestOplog { - async fn add(&self, entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + entry: OplogEntry, + ) -> Result { let mut entries = self.entries.lock().unwrap(); entries.push(entry); - OplogIndex::from_u64(entries.len() as u64) + Ok(OplogIndex::from_u64(entries.len() as u64)) } fn enqueue_add(&self, entry: OplogEntry) -> OplogAddReceipt { let mut entries = self.entries.lock().unwrap(); entries.push(entry); let index = OplogIndex::from_u64(entries.len() as u64); - Box::pin(async move { index }) + Box::pin(async move { Ok(index) }) } async fn add_pair( &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { let mut entries = self.entries.lock().unwrap(); entries.push(start); let first_idx = OplogIndex::from_u64(entries.len() as u64); entries.push(make_second(first_idx)); let second_idx = OplogIndex::from_u64(entries.len() as u64); - (first_idx, second_idx) + Ok((first_idx, second_idx)) } async fn add_start_with_reserved_raw_payload( &self, _serialized_request: Vec, _build_start: Box Result + Send>, - ) -> Result { + ) -> Result { unimplemented!() } async fn add_start_with_indexed_reserved_raw_payload( &self, _build_request: crate::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { unimplemented!() } @@ -164,8 +167,11 @@ impl Oplog for FrameTestOplog { 0 } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { - BTreeMap::new() + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, crate::services::oplog::OplogError> { + Ok(BTreeMap::new()) } async fn current_oplog_index(&self) -> OplogIndex { diff --git a/golem-worker-executor/src/durable_host/replay_state/tests.rs b/golem-worker-executor/src/durable_host/replay_state/tests.rs index 7af1c5a7c4..ed15c49b14 100644 --- a/golem-worker-executor/src/durable_host/replay_state/tests.rs +++ b/golem-worker-executor/src/durable_host/replay_state/tests.rs @@ -62,39 +62,42 @@ impl InMemoryOplog { #[async_trait] impl Oplog for InMemoryOplog { - async fn add(&self, entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + entry: OplogEntry, + ) -> Result { let mut entries = self.entries.lock().unwrap(); entries.push(entry); - OplogIndex::from_u64(entries.len() as u64) + Ok(OplogIndex::from_u64(entries.len() as u64)) } fn enqueue_add(&self, entry: OplogEntry) -> OplogAddReceipt { let mut entries = self.entries.lock().unwrap(); entries.push(entry); let index = OplogIndex::from_u64(entries.len() as u64); - Box::pin(async move { index }) + Box::pin(async move { Ok(index) }) } async fn add_pair( &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { let mut entries = self.entries.lock().unwrap(); entries.push(start); let first_idx = OplogIndex::from_u64(entries.len() as u64); entries.push(make_second(first_idx)); let second_idx = OplogIndex::from_u64(entries.len() as u64); - (first_idx, second_idx) + Ok((first_idx, second_idx)) } async fn add_start_with_reserved_raw_payload( &self, serialized_request: Vec, build_start: Box Result + Send>, - ) -> Result { + ) -> Result { let entry = build_start(RawOplogPayload::SerializedInline(serialized_request))?; - let index = self.add(entry.clone()).await; + let index = self.add(entry.clone()).await?; Ok(OrderedOplogStart { index, entry, @@ -105,7 +108,7 @@ impl Oplog for InMemoryOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: crate::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let mut entries = self.entries.lock().unwrap(); let index = OplogIndex::from_u64(entries.len() as u64 + 1); let (serialized_request, build_start) = build_request(index)?; @@ -122,8 +125,11 @@ impl Oplog for InMemoryOplog { 0 } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { - BTreeMap::new() + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, crate::services::oplog::OplogError> { + Ok(BTreeMap::new()) } async fn current_oplog_index(&self) -> OplogIndex { @@ -238,6 +244,7 @@ fn invocation_started(wallet_pin: InvocationWalletPin) -> OplogEntry { trace_states: Vec::new(), invocation_context: Vec::new(), wallet_pin: Some(wallet_pin), + shard_epoch: None, } } @@ -1060,7 +1067,7 @@ fn fork_start() -> OplogEntry { async fn replay_state_over(entries: Vec) -> ReplayState { let oplog = Arc::new(InMemoryOplog::new()); for entry in entries { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; test_replay_state(test_agent_id(), oplog, DeletedRegions::default(), None) @@ -1092,9 +1099,9 @@ async fn held_completed_reconstruction() -> ( let parent = OplogIndex::from_u64(1); let (start, identity) = rejected_tool_reconstruction_start(parent); let oplog = Arc::new(InMemoryOplog::new()); - oplog.add(noop()).await; - oplog.add(start).await; - oplog.add(end_for(2, 1)).await; + oplog.add(noop()).await.unwrap(); + oplog.add(start).await.unwrap(); + oplog.add(end_for(2, 1)).await.unwrap(); let replay = test_replay_state( test_agent_id(), oplog.clone(), @@ -1118,7 +1125,7 @@ async fn held_completed_reconstruction() -> ( #[test] async fn growing_replay_target_revokes_published_live_state() { let oplog = Arc::new(InMemoryOplog::new()); - oplog.add(noop()).await; + oplog.add(noop()).await.unwrap(); let replay = test_replay_state( test_agent_id(), oplog.clone(), @@ -1129,7 +1136,7 @@ async fn growing_replay_target_revokes_published_live_state() { .expect("failed to build replay state"); assert!(replay.is_live_published()); - let new_target = oplog.add(noop()).await; + let new_target = oplog.add(noop()).await.unwrap(); replay .set_replay_target(new_target) .await @@ -1165,7 +1172,7 @@ async fn growing_replay_target_revokes_an_active_settling_transition() { .await .expect("primary transition did not enter settling"); - let new_target = oplog.add(noop()).await; + let new_target = oplog.add(noop()).await.unwrap(); replay .set_replay_target(new_target) .await @@ -1275,10 +1282,10 @@ async fn target_growth_does_not_misclassify_a_reconstruction_as_incomplete() { let (first_start, identity) = rejected_tool_reconstruction_start(parent); let (second_start, _) = rejected_tool_reconstruction_start(parent); let oplog = Arc::new(InMemoryOplog::new()); - oplog.add(noop()).await; - oplog.add(first_start).await; - oplog.add(second_start).await; - oplog.add(end_for(3, 2)).await; + oplog.add(noop()).await.unwrap(); + oplog.add(first_start).await.unwrap(); + oplog.add(second_start).await.unwrap(); + oplog.add(end_for(3, 2)).await.unwrap(); let replay = test_replay_state( test_agent_id(), oplog.clone(), @@ -1324,7 +1331,7 @@ async fn target_growth_does_not_misclassify_a_reconstruction_as_incomplete() { "the incomplete candidate bypassed the completed reconstruction fence" ); - let new_target = oplog.add(end_for(2, 1)).await; + let new_target = oplog.add(end_for(2, 1)).await.unwrap(); replay .set_replay_target(new_target) .await @@ -1399,7 +1406,7 @@ async fn concurrent_same_target_transitions_are_idempotent() { async fn old_settler_cannot_publish_a_grown_target() { let (replay, oplog, reconstruction) = held_completed_reconstruction().await; let old_target = replay.switch_cursor_to_live().await.unwrap(); - let new_target = oplog.add(noop()).await; + let new_target = oplog.add(noop()).await.unwrap(); replay .set_replay_target(new_target) .await @@ -1449,7 +1456,7 @@ async fn old_settler_cannot_publish_a_grown_target() { #[test] async fn owner_failure_wins_when_reconstruction_barrier_is_already_empty() { let oplog = Arc::new(InMemoryOplog::new()); - oplog.add(noop()).await; + oplog.add(noop()).await.unwrap(); let owner_operations = crate::durable_host::tool::operation::OwnerToolOperations::new(); let replay = ReplayState::new_for_owner( test_agent_id(), @@ -1548,7 +1555,7 @@ async fn permission_events_replay_after_invocation_wallet_pin() { }, start_now(), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let replay_state = test_replay_state(owned_agent_id, oplog, DeletedRegions::default(), None) @@ -1675,7 +1682,7 @@ async fn permission_events_are_recovered_from_skipped_regions() { }, start_now(), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion::from_range(2..=2)]); @@ -1711,7 +1718,7 @@ async fn snapshot_prefix_suppresses_replayed_permission_events() { }, start_now(), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion::from_range(2..=2)]); @@ -1838,7 +1845,7 @@ async fn missing_start_claim_remains_divergence_while_replaying() { async fn start_claim_reports_matching_deleted_region_while_replay_continues() { let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), start_with_parent(1)] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion::from_range(2..=2)]); @@ -1868,7 +1875,7 @@ async fn assert_request_payload_failure_is_not_reclassified_as_deleted_region( start_now_with_request_payload(OplogPayload::Inline(Box::new(expected_request.clone()))), start_now_with_request_payload(failing_payload), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion::from_range(2..=2)]); @@ -1931,7 +1938,7 @@ async fn genuine_request_mismatch_still_reports_matching_deleted_region() { start_now_with_request_payload(OplogPayload::Inline(Box::new(expected_request.clone()))), start_now_with_request_payload(OplogPayload::Inline(Box::new(different_request))), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion::from_range(2..=2)]); @@ -1954,7 +1961,7 @@ async fn genuine_request_mismatch_still_reports_matching_deleted_region() { #[test] async fn request_matching_downloads_uncached_external_payloads() { let oplog = Arc::new(InMemoryOplog::new()); - oplog.add(noop()).await; + oplog.add(noop()).await.unwrap(); let first_request: HostRequest = HostRequestPollCount { count: 1 }.into(); let second_request: HostRequest = HostRequestPollCount { count: 2 }.into(); @@ -1972,7 +1979,8 @@ async fn request_matching_downloads_uncached_external_payloads() { request: Some(payload), durable_function_type: DurableFunctionType::ReadLocal, }) - .await; + .await + .unwrap(); } let oplog: Arc = oplog; @@ -3534,7 +3542,7 @@ async fn marker_in_deleted_region_delivers_end_normally() { // the still-visible End must be delivered normally. let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), end_for(2, 42), discarded_for(2)] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = @@ -3577,7 +3585,7 @@ async fn delivered_marker_with_deleted_start_is_skipped_as_orphan() { delivered_for(2), noop(), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = @@ -3609,7 +3617,7 @@ async fn duplicate_completion_discarded_markers_fail_construction() { discarded_for(2), discarded_for(2), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let err = test_replay_state(test_agent_id(), oplog, DeletedRegions::default(), None) @@ -3631,7 +3639,7 @@ async fn conflicting_completion_markers_fail_construction() { delivered_for(2), discarded_for(2), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let err = test_replay_state(test_agent_id(), oplog, DeletedRegions::default(), None) @@ -3653,7 +3661,7 @@ async fn marker_recorded_at_runtime_is_visible_to_replay() { // already-recorded marker must be idempotent, not a duplicate-marker error. let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), end_for(2, 42)] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let rs = test_replay_state( @@ -3664,7 +3672,7 @@ async fn marker_recorded_at_runtime_is_visible_to_replay() { ) .await .expect("failed to build replay state"); - let marker_idx = oplog.add(discarded_for(2)).await; + let marker_idx = oplog.add(discarded_for(2)).await.unwrap(); rs.record_discarded_completion(OplogIndex::from_u64(2), marker_idx); rs.set_replay_target(marker_idx) .await @@ -4602,7 +4610,7 @@ async fn replay_finished_emitted_when_skipped_region_reaches_target() { // jumps the cursor over the deleted tail straight to the target (4). let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), log_entry(), log_entry()] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion { @@ -5169,7 +5177,7 @@ async fn orphan_end_with_deleted_start_is_skipped() { start_now(), end_for(4, 2), ] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion { @@ -5208,7 +5216,7 @@ async fn orphan_cancelled_with_deleted_start_is_skipped() { // [NoOp(1), Start(2), Cancelled(2→3)] with deleted region [2, 2]. let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), cancelled_for(2)] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion { @@ -5235,7 +5243,7 @@ async fn positional_reader_skips_orphan_terminal() { // must consume the orphan End at 3 and return the NoOp at 4. let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), end_for(2, 1), noop()] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion { @@ -5259,7 +5267,7 @@ async fn deleted_terminal_reports_incomplete() { // [NoOp(1), Start(2), End(2→3)] with deleted region [3, 3]. let oplog = Arc::new(InMemoryOplog::new()); for entry in [noop(), start_now(), end_for(2, 1)] { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions([OplogRegion { @@ -5388,7 +5396,7 @@ async fn replay_skips_deleted_regions_fuzz() { let oplog = Arc::new(InMemoryOplog::new()); for entry in entries { - oplog.add(entry).await; + oplog.add(entry).await.unwrap(); } let oplog: Arc = oplog; let skipped = DeletedRegions::from_regions(regions.iter().map(|&(s, e)| OplogRegion { diff --git a/golem-worker-executor/src/durable_host/suspendable_wait.rs b/golem-worker-executor/src/durable_host/suspendable_wait.rs index 7465b16329..a2a0a1c021 100644 --- a/golem-worker-executor/src/durable_host/suspendable_wait.rs +++ b/golem-worker-executor/src/durable_host/suspendable_wait.rs @@ -345,7 +345,10 @@ mod tests { #[async_trait] impl Oplog for UnusedOplog { - async fn add(&self, _entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + _entry: OplogEntry, + ) -> Result { unreachable!("oplog is unused by promise waits") } @@ -357,7 +360,7 @@ mod tests { &self, _start: OplogEntry, _make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { unreachable!("oplog is unused by promise waits") } @@ -365,7 +368,10 @@ mod tests { unreachable!("oplog is unused by this test") } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, crate::services::oplog::OplogError> { unreachable!("oplog is unused by this test") } @@ -413,14 +419,14 @@ mod tests { &self, _serialized_request: Vec, _build_start: Box Result + Send>, - ) -> Result { + ) -> Result { unreachable!("oplog is unused by this test") } async fn add_start_with_indexed_reserved_raw_payload( &self, _build_request: crate::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { unreachable!("oplog is unused by this test") } } @@ -506,7 +512,10 @@ mod tests { #[async_trait] impl Oplog for StubOplog { - async fn add(&self, _entry: OplogEntry) -> OplogIndex { + async fn add( + &self, + _entry: OplogEntry, + ) -> Result { unreachable!("oplog writes are unused by wakeup scheduling") } @@ -518,7 +527,7 @@ mod tests { &self, _start: OplogEntry, _make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { unreachable!("oplog writes are unused by wakeup scheduling") } @@ -526,7 +535,10 @@ mod tests { unreachable!("oplog is unused by this test") } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, crate::services::oplog::OplogError> { unreachable!("oplog is unused by this test") } @@ -574,14 +586,14 @@ mod tests { &self, _serialized_request: Vec, _build_start: Box Result + Send>, - ) -> Result { + ) -> Result { unreachable!("oplog is unused by this test") } async fn add_start_with_indexed_reserved_raw_payload( &self, _build_request: crate::services::oplog::IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { unreachable!("oplog is unused by this test") } } diff --git a/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs index eddf69cc11..59e4eba3a2 100644 --- a/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs @@ -2881,7 +2881,7 @@ async fn finish_span_access( if is_live { worker .add_to_oplog(OplogEntry::finish_span(parent_start_index, span_id.clone())) - .await; + .await?; } else if !is_live { crate::get_oplog_entry_owned!(replay_state, OplogEntry::FinishSpan)?; } diff --git a/golem-worker-executor/src/grpc/invocation_session.rs b/golem-worker-executor/src/grpc/invocation_session.rs index a9b251a14a..3bb1fd97db 100644 --- a/golem-worker-executor/src/grpc/invocation_session.rs +++ b/golem-worker-executor/src/grpc/invocation_session.rs @@ -2555,6 +2555,11 @@ fn pre_acceptance_rejection_reason(error: &WorkerExecutorError) -> InvocationRej | WorkerExecutorError::ComponentNotFound { .. } | WorkerExecutorError::PromiseNotFound { .. } => InvocationRejectionReason::NotFound, WorkerExecutorError::InvalidAccount => InvocationRejectionReason::Unauthorized, + // A routing miss rather than a refusal. The caller has to be able to tell it apart: it + // retries these on the shard's owner, and gives up on everything it reads as a refusal. + WorkerExecutorError::InvalidShardId { .. } + | WorkerExecutorError::ShardingNotReady + | WorkerExecutorError::OplogFenced { .. } => InvocationRejectionReason::ShardingNotReady, _ => InvocationRejectionReason::Internal, } } @@ -3614,6 +3619,34 @@ mod freshness_tests { } } + #[test] + fn routing_misses_are_rejected_as_sharding_not_ready() { + // The caller retries these on the shard's owner and gives up on anything it reads as a + // refusal, so none of them may fall through to `Internal` - which is how an executor that + // had just lost its shards used to fail an invocation outright instead of redirecting it. + for error in [ + WorkerExecutorError::InvalidShardId { + shard_id: golem_common::model::ShardId::new(0), + shard_ids: Vec::new(), + }, + WorkerExecutorError::ShardingNotReady, + WorkerExecutorError::OplogFenced { + agent_id: golem_common::model::AgentId { + component_id: golem_common::model::component::ComponentId::new(), + agent_id: "fenced".to_string(), + }, + expected_epoch: 1, + actual_epoch: Some(2), + }, + ] { + assert_eq!( + pre_acceptance_rejection_reason(&error), + InvocationRejectionReason::ShardingNotReady, + "{error}" + ); + } + } + #[test] async fn inflight_committed_acceptance_wins_after_inbound_becomes_ready() { let (acceptance_committed_tx, mut acceptance_committed_rx) = diff --git a/golem-worker-executor/src/grpc/mod.rs b/golem-worker-executor/src/grpc/mod.rs index 4e92d7ec9d..76f5b2029e 100644 --- a/golem-worker-executor/src/grpc/mod.rs +++ b/golem-worker-executor/src/grpc/mod.rs @@ -38,7 +38,7 @@ use crate::services::{ pub use crate::worker::{ PERMISSION_CARD_INSTALL_RECIPIENT_MISMATCH, PERMISSION_CARD_TRANSFER_PAYLOAD_CONFLICT, }; -use crate::worker::{Worker, WorkerUpdateMode}; +use crate::worker::{RelinquishReason, Worker, WorkerUpdateMode}; use crate::workerctx::WorkerCtx; use futures::Stream; use futures::StreamExt; @@ -1038,24 +1038,22 @@ impl + UsesAllDeps + Send + Sync + return Ok(()); } - for (agent_id, worker_details) in self.active_agents().snapshot().await { - if self.shard_service().check_worker(&agent_id).is_err() - && let Some(mut await_interrupted) = worker_details - .set_interrupting(InterruptKind::Restart) - .await - { - // A closed channel means the interrupt already ran its course, - // which is all this waits for. - let _ = await_interrupted.recv().await; - } - } + // Given up, not restarted: a restart in place would reopen each agent's oplog with the + // epoch this executor no longer holds. They are dropped from here and recovered by the + // shards' new owners. + let shard_service = self.shard_service(); + self.active_agents() + .relinquish_matching(RelinquishReason::ShardRevoked, |agent_id| { + shard_service.check_worker(agent_id).is_err() + }) + .await; Ok(()) } /// Full replace: the request carries this executor's complete shard set /// with epochs and the cluster's shard count. Anything absent from the - /// set is dropped, and any agent whose shard went away is restarted. + /// set is dropped, and any agent whose shard went away is given up. async fn assign_shards_internal( &self, request: golem::workerexecutor::v1::AssignShardsRequest, @@ -1119,20 +1117,17 @@ impl + UsesAllDeps + Send + Sync + T: HasAll + Send + Sync + 'static, { let ticket = this.shard_manager_service().recovery_deferred(); - - // Pure set membership on purpose: a lapsed lease must not restart every - // running agent: a lapsed lease refuses new work and leaves running work alone. - for (agent_id, worker_details) in this.active_agents().snapshot().await { - if this.shard_service().check_worker(&agent_id).is_err() - && let Some(mut await_interrupted) = worker_details - .set_interrupting(InterruptKind::Restart) - .await - { - // A closed channel means the interrupt already ran its course, - // which is all this waits for. - let _ = await_interrupted.recv().await; - } - } + // Pure set membership on purpose: a lapsed lease must not give up every running agent - + // a lapsed lease refuses new work and leaves running work alone. + // + // Given up rather than restarted: a narrowing delivery means these shards have another + // owner now, and a restart in place would reopen their oplogs at the stale epoch. + let shard_service = this.shard_service(); + this.active_agents() + .relinquish_matching(RelinquishReason::ShardNotAssigned, |agent_id| { + shard_service.check_worker(agent_id).is_err() + }) + .await; if !this.shard_service().is_ready() { tracing::info!( diff --git a/golem-worker-executor/src/lib.rs b/golem-worker-executor/src/lib.rs index 2a3732dd0a..4641529848 100644 --- a/golem-worker-executor/src/lib.rs +++ b/golem-worker-executor/src/lib.rs @@ -109,6 +109,7 @@ use async_trait::async_trait; use futures::TryFutureExt; use golem_api_grpc::proto; use golem_api_grpc::proto::golem::workerexecutor::v1::worker_executor_server::WorkerExecutorServer; +use golem_common::SafeDisplay; use golem_common::config::DbSqliteConfig; use golem_common::redis::RedisPool; use golem_service_base::clients::registry::{GrpcRegistryService, RegistryService}; @@ -889,6 +890,12 @@ pub async fn create_worker_executor_impl< shutdown.clone(), ); + check_oplog_fencing( + shard_manager_service.requires_oplog_fencing(), + indexed_storage.as_ref(), + &golem_config.indexed_storage, + )?; + let quota_service = bootstrap.create_quota_service( shard_manager_client, &golem_config.quota_service, @@ -1300,3 +1307,80 @@ async fn build_inner_key_value_storage( } } } + +/// Refuses a configuration whose oplog writes cannot be fenced on the shard epoch. +/// +/// Shards move between executors under a real shard manager, so two executors can believe they +/// own the same agent at once; the storage fence is what stops the one that has lost the shard +/// from writing. Without it the damage is silent, which is why this is a startup failure rather +/// than a warning. +/// +/// Keyed on the services, not on the configuration: it is the `Bootstrap` override in effect - +/// not a config value - that decides whether shards can move at all, which is why the +/// single-shard executor and the debugging service are exempt without naming them here. +fn check_oplog_fencing( + requires_oplog_fencing: bool, + indexed_storage: &(dyn IndexedStorage + Send + Sync), + indexed_storage_config: &IndexedStorageConfig, +) -> anyhow::Result<()> { + if requires_oplog_fencing && !indexed_storage.supports_epoch_fencing() { + anyhow::bail!( + "The configured indexed storage cannot fence oplog writes on the shard epoch, and \ + this executor runs with a shard manager that moves shards between executors. \ + Without the fence, an executor that has lost a shard can keep writing to its \ + agents' oplogs. Set GOLEM__INDEXED_STORAGE__TYPE to one of Postgres, Sqlite, \ + KVStoreSqlite, MultiSqlite or KVStoreMultiSqlite. Configured storage: {}", + indexed_storage_config.to_safe_string(), + ); + } + Ok(()) +} + +#[cfg(test)] +mod oplog_fencing_guard_tests { + use super::*; + use crate::services::golem_config::{ + IndexedStorageInMemoryConfig, IndexedStorageMultiSqliteConfig, + }; + use crate::storage::indexed::memory::InMemoryIndexedStorage; + use crate::storage::indexed::multi_sqlite::MultiSqliteIndexedStorage; + use test_r::test; + + #[test] + fn a_non_fencing_backend_under_a_real_shard_manager_is_refused() { + let storage = InMemoryIndexedStorage::new(); + let config = IndexedStorageConfig::InMemory(IndexedStorageInMemoryConfig {}); + + let error = check_oplog_fencing(true, &storage, &config) + .expect_err("an unfenced backend with a real shard manager must refuse to start"); + let message = error.to_string(); + // The message has to name the way out, or the operator is left guessing. + assert!( + message.contains("GOLEM__INDEXED_STORAGE__TYPE"), + "the error must name the setting to change: {message}" + ); + } + + #[test] + fn the_same_backend_is_allowed_without_a_real_shard_manager() { + let storage = InMemoryIndexedStorage::new(); + let config = IndexedStorageConfig::InMemory(IndexedStorageInMemoryConfig {}); + + // Single-shard mode: nothing can take the shard away, so there is no second writer. + check_oplog_fencing(false, &storage, &config).expect("single-shard mode needs no fence"); + } + + #[test] + fn a_fencing_backend_is_allowed_either_way() { + let dir = tempfile::TempDir::new().unwrap(); + let storage = MultiSqliteIndexedStorage::new(dir.path(), 1, false); + let config = IndexedStorageConfig::MultiSqlite(IndexedStorageMultiSqliteConfig { + root_dir: dir.path().to_path_buf(), + max_connections: 1, + foreign_keys: false, + }); + + check_oplog_fencing(true, &storage, &config).expect("multi-sqlite fences"); + check_oplog_fencing(false, &storage, &config).expect("multi-sqlite fences"); + } +} diff --git a/golem-worker-executor/src/model/mod.rs b/golem-worker-executor/src/model/mod.rs index a253d422bd..a2870e8c1e 100644 --- a/golem-worker-executor/src/model/mod.rs +++ b/golem-worker-executor/src/model/mod.rs @@ -481,6 +481,13 @@ impl TrapType { Some(WorkerExecutorError::PermissionDenied { details }) => { make_error(AgentError::PermissionDenied(details.clone())) } + // Not a failure of the invocation: the storage refused the write + // because the shard has a new owner. Classified as an interrupt so + // the loop stops the agent without appending an `Error` entry to an + // oplog that is no longer this executor's to write. + Some(WorkerExecutorError::OplogFenced { .. }) => { + TrapType::Interrupt(InterruptKind::ShardLost) + } Some(WorkerExecutorError::ParamTypeMismatch { details }) => { make_error(AgentError::InvalidRequest(details.clone())) } @@ -499,7 +506,7 @@ impl TrapType { // // `WorkerExecutorError::Runtime` is intentionally NOT // mapped here: it is also used as a generic transient - // error wrapper (e.g. for `Oplog::fallible_add` + // error wrapper (e.g. for an `OplogError::Storage` // failures) and must remain retriable via the default // policy path (`AgentError::Unknown`). Some(WorkerExecutorError::UnexpectedOplogEntry { expected, got }) => { @@ -537,6 +544,10 @@ impl TrapType { TrapType::Interrupt(InterruptKind::Interrupt(_)) => Some(WorkerExecutorError::runtime( "Interrupted via the Golem API", )), + // What a caller can act on: refresh the routing table and retry on the owner. + TrapType::Interrupt(InterruptKind::ShardLost) => { + Some(WorkerExecutorError::ShardingNotReady) + } TrapType::Error { error, .. } => match error { AgentError::InvalidRequest(msg) => { Some(WorkerExecutorError::invalid_request(msg.clone())) @@ -911,6 +922,57 @@ mod tests { )); } + /// The contract every fenced host-call site depends on: a fence that escapes a host function + /// as an `anyhow` error must classify as `ShardLost`, so the loop gives the agent up instead + /// of appending an `Error` entry to the very oplog that refused the write. + #[test] + fn a_fenced_oplog_write_escaping_a_host_call_classifies_as_shard_lost() { + let fence = crate::services::oplog::OplogFence { + agent_id: golem_common::model::AgentId { + component_id: ComponentId::new(), + agent_id: "fenced-host-call".to_string(), + }, + expected_epoch: golem_common::model::ShardEpoch(7), + actual_epoch: Some(golem_common::model::ShardEpoch(8)), + }; + + let trap = TrapType::from_error::( + &anyhow::anyhow!(WorkerExecutorError::from( + crate::services::oplog::OplogError::Fenced(fence) + )), + OplogIndex::INITIAL, + false, + false, + AgentMode::Durable, + ); + + assert!( + matches!(trap, TrapType::Interrupt(InterruptKind::ShardLost)), + "a fenced write must be an interrupt, got {trap:?}" + ); + } + + /// The deliberate other half: a transient storage failure is not a fence and must stay a + /// retriable failure. Classifying it as `ShardLost` would hand an agent to another executor + /// over a blip that retrying would have cleared. + #[test] + fn a_transient_oplog_storage_failure_does_not_relinquish_the_agent() { + let trap = TrapType::from_error::( + &anyhow::anyhow!(WorkerExecutorError::from( + crate::services::oplog::OplogError::Storage("connection reset".to_string()) + )), + OplogIndex::INITIAL, + false, + false, + AgentMode::Durable, + ); + + assert!( + !matches!(trap, TrapType::Interrupt(InterruptKind::ShardLost)), + "a transient storage failure must not be treated as a lost shard, got {trap:?}" + ); + } + #[test] fn semantic_trap_retry_override_carries_retry_point() { use crate::durable_host::durability::{ @@ -988,6 +1050,44 @@ mod tests { assert_eq!(decision, Some(RetryDecision::None)); } + #[test] + fn a_fenced_oplog_write_is_a_lost_shard_and_is_never_retried() { + let agent_id = AgentId { + component_id: golem_common::model::component::ComponentId::new(), + agent_id: "fenced".to_string(), + }; + let trap = TrapType::from_worker_executor_error::( + golem_service_base::error::worker_executor::WorkerExecutorError::oplog_fenced( + agent_id, + 3, + Some(4), + ), + OplogIndex::INITIAL, + false, + false, + AgentMode::Durable, + ); + + // An interrupt, not an error: no `Error` entry may be appended to an oplog that belongs + // to another executor now. + assert!(matches!( + trap, + TrapType::Interrupt(InterruptKind::ShardLost) + )); + + // Callers are told what they can act on, which is the same thing as for a lapsed lease. + assert!(matches!( + trap.as_golem_error(""), + Some(WorkerExecutorError::ShardingNotReady) + )); + + // And it is never retried in place - that would reopen the oplog at the stale epoch. + let decision = crate::durable_host::DurableWorkerCtx::< + crate::workerctx::default::Context, + >::fixed_decision_for_trap_type(&trap); + assert_eq!(decision, Some(RetryDecision::None)); + } + #[test] fn permission_denied_is_a_non_retriable_invocation_rejection() { let trap = TrapType::from_worker_executor_error::( @@ -1024,7 +1124,7 @@ mod tests { #[test] fn runtime_error_falls_back_to_unknown_and_is_policy_retriable() { // `WorkerExecutorError::Runtime` is a generic transient-error wrapper - // (used e.g. for `Oplog::fallible_add` failures). It must not be + // (used e.g. for `OplogError::Storage` failures). It must not be // classified as `InternalError` (non-retriable); it must fall through // to `AgentError::Unknown` so the configured retry policy applies. let trap = TrapType::from_error::( diff --git a/golem-worker-executor/src/model/public_oplog/mod.rs b/golem-worker-executor/src/model/public_oplog/mod.rs index fe0570a64a..0a747c6cc5 100644 --- a/golem-worker-executor/src/model/public_oplog/mod.rs +++ b/golem-worker-executor/src/model/public_oplog/mod.rs @@ -899,6 +899,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { trace_states, invocation_context, wallet_pin, + shard_epoch: _, } => { let invocation_payload: AgentInvocationPayload = oplog_service .download_payload(owned_agent_id, agent_mode, payload) diff --git a/golem-worker-executor/src/model/public_oplog/tests.rs b/golem-worker-executor/src/model/public_oplog/tests.rs index b2ebd08b99..0b75be3f92 100644 --- a/golem-worker-executor/src/model/public_oplog/tests.rs +++ b/golem-worker-executor/src/model/public_oplog/tests.rs @@ -246,6 +246,7 @@ async fn public_oplog_zero_start_reads_from_initial_index() { make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let timestamp = Timestamp::now_utc(); @@ -255,10 +256,11 @@ async fn public_oplog_zero_start_reads_from_initial_index() { timestamp, entity_parent_start_index: None, }) - .await, + .await + .unwrap(), OplogIndex::INITIAL ); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let chunk = get_public_oplog_chunk( Arc::new(PanicComponentService), @@ -309,10 +311,11 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; - let agent_entry = oplog.add(OplogEntry::no_op(None)).await; + let agent_entry = oplog.add(OplogEntry::no_op(None)).await.unwrap(); let observational_owner = oplog .add(OplogEntry::Start { timestamp: Timestamp::now_utc(), @@ -323,7 +326,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: None, durable_function_type: DurableFunctionType::WriteLocal, }) - .await; + .await + .unwrap(); let middleware_entity = AgentEntity::ToolMiddleware(ToolMiddlewareName::try_from("audit").unwrap()); let middleware_input = "middleware-input" @@ -347,9 +351,10 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: Some(OplogPayload::Inline(Box::new(middleware_request))), durable_function_type: DurableFunctionType::WriteLocal, }) - .await; + .await + .unwrap(); - let interleaved_agent_entry = oplog.add(OplogEntry::no_op(None)).await; + let interleaved_agent_entry = oplog.add(OplogEntry::no_op(None)).await.unwrap(); let child_start = oplog .add(OplogEntry::Start { timestamp: Timestamp::now_utc(), @@ -360,7 +365,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: Some(OplogPayload::Inline(Box::new(HostRequestNoInput {}.into()))), durable_function_type: DurableFunctionType::ReadLocal, }) - .await; + .await + .unwrap(); let tool_entity = AgentEntity::Tool(ToolName::try_from("lookup").unwrap()); let secret_id = Uuid::from_u128(1); @@ -400,7 +406,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: Some(OplogPayload::Inline(Box::new(tool_request))), durable_function_type: DurableFunctionType::WriteLocal, }) - .await; + .await + .unwrap(); let entity_retry_error = oplog .add(OplogEntry::error( Some(tool_start), @@ -409,8 +416,12 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { false, None, )) - .await; - let entity_marker = oplog.add(OplogEntry::no_op(Some(tool_start))).await; + .await + .unwrap(); + let entity_marker = oplog + .add(OplogEntry::no_op(Some(tool_start))) + .await + .unwrap(); let log_index = oplog .add(OplogEntry::Log { timestamp: Timestamp::now_utc(), @@ -419,7 +430,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { context: "tool".to_string(), message: "entity-attribution-needle".to_string(), }) - .await; + .await + .unwrap(); let span_id = SpanId::generate(); let span_index = oplog .add(OplogEntry::StartSpan { @@ -430,7 +442,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { linked_context_id: None, attributes: AttributeMap(HashMap::new()), }) - .await; + .await + .unwrap(); let stream_frame_index = oplog .add(OplogEntry::HostStreamFrame { timestamp: Timestamp::now_utc(), @@ -438,7 +451,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { kind: HostStreamKind::P3HttpRequestBody, payload: OplogPayload::Inline(Box::new(HostRequestNoInput {}.into())), }) - .await; + .await + .unwrap(); let reveal_secret_id = Uuid::from_u128(2); let reveal_request = HostRequestSecretReveal { @@ -488,7 +502,8 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: None, durable_function_type: DurableFunctionType::ReadLocal, }) - .await; + .await + .unwrap(); let observational_log = oplog .add(OplogEntry::Log { timestamp: Timestamp::now_utc(), @@ -497,10 +512,12 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { context: "custom".to_string(), message: "agent-owned observation".to_string(), }) - .await; + .await + .unwrap(); let observational_end = oplog .add(OplogEntry::end(observational_start, None, false)) - .await; + .await + .unwrap(); let transaction_start = oplog .add(OplogEntry::Start { @@ -512,24 +529,31 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: None, durable_function_type: DurableFunctionType::WriteRemoteTransaction(None), }) - .await; + .await + .unwrap(); let transaction_begin = oplog .add(OplogEntry::BeginRemoteTransaction { timestamp: Timestamp::now_utc(), transaction_id: TransactionId::new("entity-transaction".to_string()), original_begin_index: None, }) - .await; + .await + .unwrap(); let transaction_commit = oplog .add(OplogEntry::CommittedRemoteTransaction { timestamp: Timestamp::now_utc(), begin_index: transaction_start, }) - .await; + .await + .unwrap(); let transaction_end = oplog .add(OplogEntry::end(transaction_start, None, false)) - .await; - let child_end = oplog.add(OplogEntry::end(child_start, None, false)).await; + .await + .unwrap(); + let child_end = oplog + .add(OplogEntry::end(child_start, None, false)) + .await + .unwrap(); let tool_terminal = SerializableToolOperationTerminal { body_execution: SerializableEntityBodyExecution::Executed, result: Ok(SerializableToolStructuredResult { result: None }), @@ -546,10 +570,12 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { Some(OplogPayload::Inline(Box::new(tool_response))), false, )) - .await; + .await + .unwrap(); let completion = oplog .add(OplogEntry::completion_delivered(tool_start)) - .await; + .await + .unwrap(); let rejected_request: HostRequest = HostRequestGolemToolInvocationRejected { attempt_ordinal: 0, @@ -573,13 +599,16 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { request: Some(OplogPayload::Inline(Box::new(rejected_request))), durable_function_type: DurableFunctionType::WriteLocal, }) - .await; + .await + .unwrap(); let rejected_end = oplog .add(OplogEntry::end(rejected_start, None, false)) - .await; + .await + .unwrap(); let middleware_end = oplog .add(OplogEntry::end(middleware_start, None, false)) - .await; + .await + .unwrap(); let final_log = oplog .add(OplogEntry::Log { timestamp: Timestamp::now_utc(), @@ -588,8 +617,9 @@ async fn entity_attribution_is_nested_page_independent_and_order_preserving() { context: "tool".to_string(), message: "last-entity-attribution-needle".to_string(), }) - .await; - oplog.commit(CommitLevel::Always).await; + .await + .unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); let components: Arc = Arc::new(PanicComponentService); let chunk = get_public_oplog_chunk( @@ -832,10 +862,11 @@ async fn explicit_entity_attribution_rejects_non_causal_and_non_entity_anchors() make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; - let non_start = oplog.add(OplogEntry::no_op(None)).await; + let non_start = oplog.add(OplogEntry::no_op(None)).await.unwrap(); let non_entity_start = oplog .add(OplogEntry::Start { timestamp: Timestamp::now_utc(), @@ -846,7 +877,8 @@ async fn explicit_entity_attribution_rejects_non_causal_and_non_entity_anchors() request: None, durable_function_type: DurableFunctionType::WriteLocal, }) - .await; + .await + .unwrap(); let entity_request = test_entity_request( &owned_agent_id, AgentEntity::Tool(ToolName::try_from("valid").unwrap()), @@ -864,16 +896,24 @@ async fn explicit_entity_attribution_rejects_non_causal_and_non_entity_anchors() request: Some(OplogPayload::Inline(Box::new(entity_request))), durable_function_type: DurableFunctionType::WriteLocal, }) - .await; - let valid = oplog.add(OplogEntry::no_op(Some(entity_start))).await; - let invalid_non_start = oplog.add(OplogEntry::no_op(Some(non_start))).await; - let invalid_non_entity = oplog.add(OplogEntry::no_op(Some(non_entity_start))).await; + .await + .unwrap(); + let valid = oplog + .add(OplogEntry::no_op(Some(entity_start))) + .await + .unwrap(); + let invalid_non_start = oplog.add(OplogEntry::no_op(Some(non_start))).await.unwrap(); + let invalid_non_entity = oplog + .add(OplogEntry::no_op(Some(non_entity_start))) + .await + .unwrap(); let invalid_forward_index = invalid_non_entity.next(); let future_entity_start = invalid_forward_index.next(); assert_eq!( oplog .add(OplogEntry::no_op(Some(future_entity_start))) - .await, + .await + .unwrap(), invalid_forward_index ); assert_eq!( @@ -887,10 +927,11 @@ async fn explicit_entity_attribution_rejects_non_causal_and_non_entity_anchors() request: None, durable_function_type: DurableFunctionType::WriteLocal, }) - .await, + .await + .unwrap(), future_entity_start ); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let components: Arc = Arc::new(PanicComponentService); let valid_chunk = get_public_oplog_chunk( @@ -975,6 +1016,7 @@ async fn p3_payloads_render_through_public_oplog_api_and_wit() { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -1196,7 +1238,8 @@ async fn p3_payloads_render_through_public_oplog_api_and_wit() { request: Some(cancelled_request_payload), durable_function_type: DurableFunctionType::WriteRemote, }) - .await; + .await + .unwrap(); expected_starts.insert( cancelled_start_index, ( @@ -1219,8 +1262,9 @@ async fn p3_payloads_render_through_public_oplog_api_and_wit() { cancelled_start_index, Some(partial_payload), )) - .await; - oplog.commit(CommitLevel::Always).await; + .await + .unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); let last_index = oplog_service .get_last_index(&owned_agent_id, AgentMode::Durable) diff --git a/golem-worker-executor/src/model/public_oplog/wit.rs b/golem-worker-executor/src/model/public_oplog/wit.rs index d844b23e6c..cc656321e7 100644 --- a/golem-worker-executor/src/model/public_oplog/wit.rs +++ b/golem-worker-executor/src/model/public_oplog/wit.rs @@ -1219,6 +1219,8 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { trace_states: params.trace_states, invocation_context, wallet_pin: None, + // Raw-only, and absent from the WIT record by design. + shard_epoch: None, }) } oplog::OplogEntry::AgentInvocationFinished(params) => { @@ -1996,6 +1998,7 @@ impl TryFrom for oplog::OplogEntry { trace_states, invocation_context, wallet_pin: _, + shard_epoch: _, } => Ok(Self::AgentInvocationStarted( oplog::RawAgentInvocationStartedParameters { timestamp: timestamp.into(), @@ -2548,6 +2551,7 @@ mod tests { pinned_card_ids: pinned_card_ids.clone(), scope_card_id: Some(scope_card_id), }), + shard_epoch: None, }; let encoded = oplog::OplogEntry::try_from(raw_entry).unwrap(); diff --git a/golem-worker-executor/src/services/active_agents/memory_probe.rs b/golem-worker-executor/src/services/active_agents/memory_probe.rs index 6b150d4d68..42ae368846 100644 --- a/golem-worker-executor/src/services/active_agents/memory_probe.rs +++ b/golem-worker-executor/src/services/active_agents/memory_probe.rs @@ -383,6 +383,12 @@ mod tests { } } + /// These tests wait for a refresh that runs on `spawn_blocking`, so what they are really + /// bounded by is a blocking-pool slot becoming free, not the refresh being quick. Under the + /// full parallel lib suite that can take seconds. The deadline exists to fail rather than + /// hang; no assertion depends on its value. + const REFRESH_COMPLETION_TIMEOUT: Duration = Duration::from_secs(30); + #[derive(Debug)] struct PanickingProbe { reads: Arc, @@ -437,7 +443,7 @@ mod tests { current_bytes.store(42, Ordering::Relaxed); assert_eq!(probe.snapshot().current_bytes, 1); - tokio::time::timeout(Duration::from_secs(1), async { + tokio::time::timeout(REFRESH_COMPLETION_TIMEOUT, async { while reads.load(Ordering::Acquire) != 2 { tokio::task::yield_now().await; } @@ -455,7 +461,7 @@ mod tests { *refresh_gate.0.lock().unwrap() = true; refresh_gate.1.notify_one(); - tokio::time::timeout(Duration::from_secs(1), async { + tokio::time::timeout(REFRESH_COMPLETION_TIMEOUT, async { while probe.snapshot().limit_bytes != 50 || probe.snapshot().current_bytes != 42 { tokio::task::yield_now().await; } @@ -485,7 +491,7 @@ mod tests { tokio::time::sleep(refresh_interval).await; assert_eq!(probe.snapshot().current_bytes, 1); - tokio::time::timeout(Duration::from_secs(1), async { + tokio::time::timeout(REFRESH_COMPLETION_TIMEOUT, async { while reads.load(Ordering::Acquire) != 2 { tokio::task::yield_now().await; } @@ -494,7 +500,7 @@ mod tests { .unwrap(); tokio::time::sleep(refresh_interval).await; - tokio::time::timeout(Duration::from_secs(1), async { + tokio::time::timeout(REFRESH_COMPLETION_TIMEOUT, async { while probe.snapshot().current_bytes != 42 { tokio::task::yield_now().await; } @@ -519,7 +525,7 @@ mod tests { ); assert_eq!(probe.snapshot().current_bytes, 1); - tokio::time::timeout(Duration::from_secs(1), async { + tokio::time::timeout(REFRESH_COMPLETION_TIMEOUT, async { while probe.inner.refresh_in_progress.load(Ordering::Acquire) { tokio::task::yield_now().await; } diff --git a/golem-worker-executor/src/services/active_agents/mod.rs b/golem-worker-executor/src/services/active_agents/mod.rs index 80bb83a90e..925ebe7925 100644 --- a/golem-worker-executor/src/services/active_agents/mod.rs +++ b/golem-worker-executor/src/services/active_agents/mod.rs @@ -62,7 +62,8 @@ use crate::worker::instance::{ use crate::worker::owner_lane::{EntityCallMode, OwnerInvocationId}; use crate::worker::status_flusher::AgentStatusFlushQueue; use crate::worker::{ - EvictionClass, EvictionStopOutcome, FilesystemPressureEligibility, UnloadRequest, + EvictionClass, EvictionStopOutcome, FilesystemPressureEligibility, RelinquishReason, + UnloadRequest, }; use crate::workerctx::WorkerCtx; use golem_common::cache::{BackgroundEvictionMode, Cache, FullCacheEvictionMode, SimpleCache}; @@ -800,12 +801,23 @@ impl ActiveAgents { } pub async fn remove(&self, owned_agent_id: &OwnedAgentId) { + self.remove_with( + owned_agent_id, + OwnerFailureWinner::Lifecycle(InterruptKind::Interrupt(Timestamp::now_utc())), + ) + .await + } + + /// [`Self::remove`] with an explicit reason for tearing the agent's entity bodies down. + /// A relinquished agent must not report itself as interrupted through the Golem API: it was + /// not, its shard moved. + pub(crate) async fn remove_with( + &self, + owned_agent_id: &OwnedAgentId, + owner_failure: OwnerFailureWinner, + ) { if let Some(active_agent) = self.agents.get(owned_agent_id).await { - active_agent - .fence_entity_bodies(OwnerFailureWinner::Lifecycle(InterruptKind::Interrupt( - Timestamp::now_utc(), - ))) - .await; + active_agent.fence_entity_bodies(owner_failure).await; let worker = active_agent.primary(); self.card_interest_index .set_card_interest(worker.owned_agent_id().clone(), &[]) @@ -869,6 +881,37 @@ impl ActiveAgents { .collect() } + /// Gives up every agent the predicate selects: stops each one here and drops it from this + /// executor, so the shard's new owner recovers it. + /// + /// Concurrent rather than sequential, unlike [`Self::unload_environment`]: a revoke can name + /// many agents and each stop waits for that agent's invocation loop to exit. No acknowledgement + /// channel is awaited either - [`Worker::relinquish`] never subscribes to one - so an agent + /// that is already stopping cannot panic the sweep, which is what the old + /// `set_interrupting(..).recv().await.unwrap()` shape risked. + /// + /// The snapshot includes suspended, loading and already-stopping agents; the stop state + /// machine has an arm for each, so none is skipped. + pub(crate) async fn relinquish_matching( + &self, + reason: RelinquishReason, + select: impl Fn(&AgentId) -> bool, + ) { + let selected: Vec>> = self + .snapshot() + .await + .into_iter() + .filter(|(agent_id, _)| select(agent_id)) + .map(|(_, worker)| worker) + .collect(); + + futures::future::join_all(selected.into_iter().map(|worker| { + let reason = reason.clone(); + async move { worker.relinquish(reason).await } + })) + .await; + } + /// Interrupts and unloads all in-memory workers whose environment matches /// `environment_id`. Called when the environment is deleted so that /// running workers stop promptly. diff --git a/golem-worker-executor/src/services/oplog/compressed.rs b/golem-worker-executor/src/services/oplog/compressed.rs index 473c6aa8f2..f1465a58af 100644 --- a/golem-worker-executor/src/services/oplog/compressed.rs +++ b/golem-worker-executor/src/services/oplog/compressed.rs @@ -504,7 +504,7 @@ impl OplogArchive for CompressedOplogArchive { let chunk = compressed_chunk.clone(); async move { is.with_entity("compressed_oplog", "append", "compressed_entry") - .append(ns, &key, last_id_val, &chunk) + .append(ns, &key, last_id_val, &chunk, None) .await } }) diff --git a/golem-worker-executor/src/services/oplog/ephemeral.rs b/golem-worker-executor/src/services/oplog/ephemeral.rs index 0189057f26..8813b4ca95 100644 --- a/golem-worker-executor/src/services/oplog/ephemeral.rs +++ b/golem-worker-executor/src/services/oplog/ephemeral.rs @@ -22,7 +22,8 @@ use crate::services::oplog::reader::{ }; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, Oplog, OplogAddReceipt, - OplogService, OrderedOplogStart, PendingUpload, ReservedRawStartBuilder, downcast_oplog, + OplogError, OplogService, OrderedOplogStart, PendingUpload, ReservedRawStartBuilder, + downcast_oplog, }; use async_trait::async_trait; use golem_common::model::agent::AgentMode; @@ -628,18 +629,18 @@ impl Oplog for EphemeralOplog { } let owned_agent_id = self.owned_agent_id.clone(); Box::pin(async move { - done_rx.await.unwrap_or_else(|_| { + Ok(done_rx.await.unwrap_or_else(|_| { panic!( "Ephemeral oplog actor for {owned_agent_id:?} dropped an add request without replying" ) - }) + })) }) } async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { record_oplog_call("add_durable_stream_batch"); Ok(self .run_job(|done| EphemeralJob::AddDurableStreamBatch { make_batch, done }) @@ -650,21 +651,22 @@ impl Oplog for EphemeralOplog { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), OplogError> { record_oplog_call("add_pair"); - self.run_job(|done| EphemeralJob::AddPair { - start, - make_second, - done, - }) - .await + Ok(self + .run_job(|done| EphemeralJob::AddPair { + start, + make_second, + done, + }) + .await) } async fn add_start_with_reserved_raw_payload( &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result { + ) -> Result { record_oplog_call("add_start_with_reserved_raw_payload"); // Ephemeral oplogs are never replayed, so cross-call `Start` ordering need not be // deterministic and there is no deferred-upload/commit-barrier machinery here. Upload the @@ -688,13 +690,14 @@ impl Oplog for EphemeralOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { record_oplog_call("add_start_with_indexed_reserved_raw_payload"); self.run_job(|done| EphemeralJob::AddIndexedStart { build_request, done, }) .await + .map_err(OplogError::from) } async fn drop_prefix(&self, last_dropped_id: OplogIndex) -> u64 { @@ -706,11 +709,14 @@ impl Oplog for EphemeralOplog { dropped } - async fn commit(&self, level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + level: CommitLevel, + ) -> Result, OplogError> { record_oplog_call("commit"); match level { - CommitLevel::Always => self.run_job(|done| EphemeralJob::Commit { done }).await, - CommitLevel::DurableOnly => BTreeMap::new(), + CommitLevel::Always => Ok(self.run_job(|done| EphemeralJob::Commit { done }).await), + CommitLevel::DurableOnly => Ok(BTreeMap::new()), } } diff --git a/golem-worker-executor/src/services/oplog/mod.rs b/golem-worker-executor/src/services/oplog/mod.rs index 1e0ab495fa..01f6b0c322 100644 --- a/golem-worker-executor/src/services/oplog/mod.rs +++ b/golem-worker-executor/src/services/oplog/mod.rs @@ -35,7 +35,7 @@ use golem_common::model::oplog::{ }; use golem_common::model::{ AgentId, AgentInvocation, AgentInvocationResult, AgentMetadata, AgentStatusRecord, - DurableStreamSessionStatus, OwnedAgentId, ScanCursor, Timestamp, + DurableStreamSessionStatus, OwnedAgentId, ScanCursor, ShardEpoch, Timestamp, }; use golem_common::read_only_lock; use golem_common::serialization::serialize; @@ -46,7 +46,7 @@ pub use multilayer::{MultiLayerOplog, MultiLayerOplogService, OplogArchiveServic pub use primary::PrimaryOplogService; use std::any::{Any, TypeId}; use std::collections::BTreeMap; -use std::fmt::{Debug, Formatter}; +use std::fmt::{Debug, Display, Formatter}; use std::marker::PhantomData; use std::ops::Deref; use std::sync::atomic::{AtomicBool, Ordering}; @@ -103,6 +103,7 @@ pub trait OplogService: Debug + Send + Sync { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc; /// Creates an oplog whose absence has already been established by the caller. @@ -118,6 +119,7 @@ pub trait OplogService: Debug + Send + Sync { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc; /// Opens an existing oplog for the given worker. @@ -137,6 +139,7 @@ pub trait OplogService: Debug + Send + Sync { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc; async fn get_last_index( @@ -489,7 +492,69 @@ pub type IndexedReservedStartBuilder = /// A single oplog append that has already been synchronously enqueued in the oplog's ordering /// domain. Creating this receipt reserves the entry's position; awaiting it returns the assigned /// index after the append finishes. -pub type OplogAddReceipt = BoxFuture<'static, OplogIndex>; +/// Why an oplog write was refused by the storage: the shard epoch this executor asserted is +/// behind the one recorded for the oplog, because another executor owns the shard now. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OplogFence { + pub agent_id: AgentId, + pub expected_epoch: ShardEpoch, + pub actual_epoch: Option, +} + +/// The one way an oplog write can fail without taking the executor down. +/// +/// A `Fenced` write is not a storage failure - the storage is healthy and refused the write on +/// purpose - so it is returned rather than retried or panicked on, and the worker that hit it is +/// stopped and left to the shard's new owner. Every other storage failure keeps its fail-stop +/// semantics inside the oplog implementation; `Storage` exists so that test doubles and payload +/// helpers that already return a `String` can flow through the same `Result` without a second +/// error type at every call site. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OplogError { + Fenced(OplogFence), + Storage(String), +} + +impl From for OplogError { + fn from(details: String) -> Self { + OplogError::Storage(details) + } +} + +impl From for WorkerExecutorError { + fn from(error: OplogError) -> Self { + match error { + OplogError::Fenced(fence) => WorkerExecutorError::oplog_fenced( + fence.agent_id, + fence.expected_epoch.0, + fence.actual_epoch.map(|epoch| epoch.0), + ), + OplogError::Storage(details) => WorkerExecutorError::runtime(details), + } + } +} + +impl Display for OplogError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + OplogError::Fenced(fence) => write!( + f, + "oplog write for {} fenced: asserted shard epoch {}, stored {}", + fence.agent_id, + fence.expected_epoch, + fence + .actual_epoch + .map(|epoch| epoch.to_string()) + .unwrap_or_else(|| "none".to_string()) + ), + OplogError::Storage(details) => write!(f, "oplog storage error: {details}"), + } + } +} + +impl std::error::Error for OplogError {} + +pub type OplogAddReceipt = BoxFuture<'static, Result>; #[derive(Clone, Debug, PartialEq, Eq)] pub struct RawDurableStreamSessionStatus { @@ -501,7 +566,7 @@ pub struct RawDurableStreamSessionStatus { #[async_trait] pub trait Oplog: Any + Debug + Send + Sync { /// Adds a single entry to the oplog (possibly buffered), and returns its index - async fn add(&self, entry: OplogEntry) -> OplogIndex { + async fn add(&self, entry: OplogEntry) -> Result { self.enqueue_add(entry).await } @@ -521,7 +586,7 @@ pub trait Oplog: Any + Debug + Send + Sync { async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { let first_index = self.current_oplog_index().await.next(); let records = make_batch(first_index); let mut result = Vec::with_capacity(records.len()); @@ -532,7 +597,7 @@ pub trait Oplog: Any + Debug + Send + Sync { index.next() }); let entry = record.into_inline_entry(); - let index = self.add(entry.clone()).await; + let index = self.add(entry.clone()).await?; assert_eq!( index, expected_index, "oplog add_durable_stream_batch default observed a concurrent writer" @@ -542,12 +607,6 @@ pub trait Oplog: Any + Debug + Send + Sync { Ok(result) } - /// A variant of add that can inject failures in tests. TO BE REMOVED - async fn fallible_add(&self, entry: OplogEntry) -> Result<(), String> { - self.add(entry).await; - Ok(()) - } - /// Drop a chunk of entries from the beginning of the oplog /// /// This should only be called _after_ `append` succeeded in the layer below this one @@ -556,7 +615,10 @@ pub trait Oplog: Any + Debug + Send + Sync { async fn drop_prefix(&self, last_dropped_id: OplogIndex) -> u64; /// Commits the buffered entries to the oplog - async fn commit(&self, level: CommitLevel) -> BTreeMap; + async fn commit( + &self, + level: CommitLevel, + ) -> Result, OplogError>; /// Returns the current oplog index async fn current_oplog_index(&self) -> OplogIndex; @@ -618,10 +680,10 @@ pub trait Oplog: Any + Debug + Send + Sync { async fn length(&self) -> u64; /// Adds an entry to the oplog and immediately commits it - async fn add_and_commit(&self, entry: OplogEntry) -> OplogIndex { - let index = self.add(entry).await; - self.commit(CommitLevel::Always).await; - index + async fn add_and_commit(&self, entry: OplogEntry) -> Result { + let index = self.add(entry).await?; + self.commit(CommitLevel::Always).await?; + Ok(index) } /// Uploads a big oplog payload and returns a reference to it @@ -672,7 +734,7 @@ pub trait Oplog: Any + Debug + Send + Sync { &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result; + ) -> Result; /// Like [`Self::add_start_with_reserved_raw_payload`], but builds the request after the leaf /// oplog has assigned the exact `Start` index. The leaf must invoke `build_request` and append @@ -681,7 +743,7 @@ pub trait Oplog: Any + Debug + Send + Sync { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result; + ) -> Result; /// Atomically appends a `Start` entry and a second entry (its `End` or /// `Cancelled`) that references the `Start`'s `OplogIndex`. @@ -702,19 +764,21 @@ pub trait Oplog: Any + Debug + Send + Sync { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex); + ) -> Result<(OplogIndex, OplogIndex), OplogError>; - /// Like [`add_pair`](Self::add_pair) but for two already-built entries, returning a - /// `Result` so test wrappers can inject a write failure on either entry. The default - /// delegates to `add_pair`, inheriting its atomic buffering, so the two entries are - /// never split by a commit-threshold check or a crash boundary. - async fn fallible_add_pair( - &self, - first: OplogEntry, - second: OplogEntry, - ) -> Result<(OplogIndex, OplogIndex), String> { - let (first_idx, second_idx) = self.add_pair(first, Box::new(move |_| second)).await; - Ok((first_idx, second_idx)) + /// The shard epoch this oplog's writes assert, or `None` for an oplog nothing fences - one + /// opened without an ownership claim, or an ephemeral one. + /// + /// Only the primary oplog knows it, so a wrapper answers from the oplog it wraps. + fn shard_epoch(&self) -> Option { + self.inner().and_then(|inner| inner.shard_epoch()) + } + + /// The refusal this oplog has latched, if the storage has turned one of its writes away: + /// every later write fails on it, so the handle is finished. Answered without a round trip, + /// so the open-oplog cache can decline to hand a finished handle to a new opener. + fn fence(&self) -> Option { + self.inner().and_then(|inner| inner.fence()) } /// Returns the inner oplog wrapped by this implementation, if any. @@ -819,7 +883,7 @@ pub trait OplogOps: Oplog { &self, request: T, build_start: impl FnOnce(OplogPayload) -> OplogEntry + Send + 'static, - ) -> Result<(OplogIndex, PendingUpload), String> + ) -> Result<(OplogIndex, PendingUpload), OplogError> where T: BinaryCodec + Debug + Clone + PartialEq + Send + Sync + 'static, { @@ -844,7 +908,7 @@ pub trait OplogOps: Oplog { &self, build_request: impl FnOnce(OplogIndex) -> Result + Send + 'static, build_start: impl FnOnce(OplogPayload) -> OplogEntry + Send + 'static, - ) -> Result<(OplogIndex, PendingUpload), String> + ) -> Result<(OplogIndex, PendingUpload), OplogError> where T: BinaryCodec + Debug + Clone + PartialEq + Send + Sync + 'static, { @@ -891,7 +955,7 @@ pub trait OplogOps: Oplog { response: &HostResponse, function_type: DurableFunctionType, parent_start_index: Option, - ) -> Result<(OplogIndex, OplogIndex), String> { + ) -> Result<(OplogIndex, OplogIndex), OplogError> { let request_payload: OplogPayload = self.upload_payload(request).await?; let response_payload: OplogPayload = self.upload_payload(response).await?; let now = Timestamp::now_utc(); @@ -914,7 +978,7 @@ pub trait OplogOps: Oplog { forced_commit: false, }), ) - .await; + .await?; Ok((start_idx, end_idx)) } @@ -922,11 +986,11 @@ pub trait OplogOps: Oplog { &self, invocation: AgentInvocation, wallet_pin: InvocationWalletPin, - ) -> Result { + ) -> Result { let entry = self .agent_invocation_started_entry(invocation, wallet_pin) .await?; - self.add(entry.clone()).await; + self.add(entry.clone()).await?; Ok(entry) } @@ -934,11 +998,11 @@ pub trait OplogOps: Oplog { &self, invocation: AgentInvocation, wallet_pin: InvocationWalletPin, - ) -> Result { + ) -> Result { let entry = self .agent_invocation_started_entry(invocation, wallet_pin) .await?; - Ok(self.add(entry).await) + self.add(entry).await } async fn agent_invocation_started_entry( @@ -957,6 +1021,7 @@ pub trait OplogOps: Oplog { trace_states: ctx.trace_states, invocation_context, wallet_pin: Some(wallet_pin), + shard_epoch: self.shard_epoch().map(|epoch| epoch.0), }) } @@ -966,7 +1031,7 @@ pub trait OplogOps: Oplog { method_name: Option, consumed_fuel: u64, component_revision: ComponentRevision, - ) -> Result { + ) -> Result { let consumed_fuel = if consumed_fuel > i64::MAX as u64 { i64::MAX } else { @@ -981,7 +1046,7 @@ pub trait OplogOps: Oplog { consumed_fuel, component_revision, }; - self.add(entry.clone()).await; + self.add(entry.clone()).await?; Ok(entry) } @@ -1075,13 +1140,17 @@ impl OplogServiceOps for O {} struct OpenOplogEntry { pub oplog: Weak, pub initial: Arc, + /// Identifies this insertion, so that the remover the oplog runs when it is dropped removes + /// this entry and not a replacement cached under the same agent after it. + pub token: Arc<()>, } impl OpenOplogEntry { - pub fn new(oplog: Arc) -> Self { + pub fn new(oplog: Arc, token: Arc<()>) -> Self { Self { oplog: Arc::downgrade(&oplog), initial: Arc::new(AtomicBool::new(true)), + token, } } } @@ -1110,7 +1179,17 @@ impl OpenOplogs { ) -> Arc { loop { let constructor_clone = constructor.clone(); - let close = Box::new(self.oplogs.create_weak_remover(agent_id.clone())); + let token = Arc::new(()); + let close = { + let token = token.clone(); + Box::new( + self.oplogs + .create_weak_remover_if(agent_id.clone(), move |entry: &OpenOplogEntry| { + Arc::ptr_eq(&entry.token, &token) + }), + ) + }; + let entry_token = token.clone(); let entry = self .oplogs @@ -1127,13 +1206,14 @@ impl OpenOplogs { Arc::increment_strong_count(ptr); Arc::from_raw(ptr) }; - Ok(OpenOplogEntry::new(result)) + Ok(OpenOplogEntry::new(result, entry_token)) }, ) .await .unwrap(); if let Some(oplog) = entry.oplog.upgrade() { - let oplog = if entry.initial.swap(false, Ordering::AcqRel) { + let just_constructed = entry.initial.swap(false, Ordering::AcqRel); + let oplog = if just_constructed { unsafe { let ptr = Arc::into_raw(oplog); Arc::decrement_strong_count(ptr); @@ -1143,6 +1223,24 @@ impl OpenOplogs { oplog }; + // A handle the storage has fenced is finished: every write through it is refused. + // It stays alive while the worker that hit the fence is still stopping, and an + // opener arriving in that window - this executor re-granted the shard, recovering + // the agent - must get a fresh handle at its own epoch, not the finished one. + // + // Only a cache hit is discarded. An oplog refused at open is born fenced, and that + // is what its opener asked for: it is handed back so that its writes are refused, + // rather than constructed again and again. + if !just_constructed && oplog.fence().is_some() { + // Removed by its own token, so a replacement cached meanwhile is left alone. + self.oplogs + .remove_if_cached(agent_id, |cached| { + Arc::ptr_eq(&cached.token, &entry.token) + }) + .await; + continue; + } + break oplog; } else { self.oplogs.remove(agent_id).await; diff --git a/golem-worker-executor/src/services/oplog/multilayer.rs b/golem-worker-executor/src/services/oplog/multilayer.rs index aed8e79a00..b22c04a74e 100644 --- a/golem-worker-executor/src/services/oplog/multilayer.rs +++ b/golem-worker-executor/src/services/oplog/multilayer.rs @@ -24,10 +24,11 @@ use crate::services::oplog::multilayer::BackgroundTransferMessage::{ use crate::services::oplog::reader::{OplogRead, OplogReadError, OplogReadSource, fail_stop}; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, OpenOplogs, Oplog, - OplogAddReceipt, OplogConstructor, OplogService, OrderedOplogStart, ReservedRawStartBuilder, - downcast_oplog, scan_modes, + OplogAddReceipt, OplogConstructor, OplogError, OplogService, OrderedOplogStart, + ReservedRawStartBuilder, downcast_oplog, scan_modes, }; use async_trait::async_trait; +use golem_common::model::ShardEpoch; use golem_common::model::account::AccountId; use golem_common::model::agent::AgentMode; use golem_common::model::component::ComponentId; @@ -386,6 +387,7 @@ struct CreateOplogConstructor { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, } impl CreateOplogConstructor { @@ -401,6 +403,7 @@ impl CreateOplogConstructor { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Self { Self { owned_agent_id, @@ -413,6 +416,7 @@ impl CreateOplogConstructor { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, } } } @@ -444,6 +448,7 @@ impl OplogConstructor for CreateOplogConstructor { self.initial_worker_metadata, self.last_known_status, self.execution_status, + self.shard_epoch, ) .await } else { @@ -455,6 +460,7 @@ impl OplogConstructor for CreateOplogConstructor { self.initial_worker_metadata, self.last_known_status, self.execution_status, + self.shard_epoch, ) .await } @@ -467,6 +473,7 @@ impl OplogConstructor for CreateOplogConstructor { self.initial_worker_metadata, self.last_known_status, self.execution_status, + self.shard_epoch, ) .await }; @@ -551,6 +558,7 @@ impl OplogService for MultiLayerOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( @@ -566,6 +574,7 @@ impl OplogService for MultiLayerOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ), ) .await @@ -579,6 +588,7 @@ impl OplogService for MultiLayerOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( @@ -594,6 +604,7 @@ impl OplogService for MultiLayerOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ), ) .await @@ -607,6 +618,7 @@ impl OplogService for MultiLayerOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( @@ -622,6 +634,7 @@ impl OplogService for MultiLayerOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ), ) .await @@ -1119,7 +1132,7 @@ impl Oplog for MultiLayerOplog { async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { self.primary.add_durable_stream_batch(make_batch).await } @@ -1129,8 +1142,11 @@ impl Oplog for MultiLayerOplog { dropped_entries } - async fn commit(&self, level: CommitLevel) -> BTreeMap { - let result = self.primary.commit(level).await; + async fn commit( + &self, + level: CommitLevel, + ) -> Result, OplogError> { + let result = self.primary.commit(level).await?; if let Some(index) = result.keys().next_back() { self.last_reported_commit_index.max(*index); @@ -1150,7 +1166,7 @@ impl Oplog for MultiLayerOplog { }); self.last_transfer_point.max(last_committed_idx); } - result + Ok(result) } async fn current_oplog_index(&self) -> OplogIndex { @@ -1220,7 +1236,7 @@ impl Oplog for MultiLayerOplog { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), OplogError> { self.primary.add_pair(start, make_second).await } @@ -1228,7 +1244,7 @@ impl Oplog for MultiLayerOplog { &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result { + ) -> Result { self.primary .add_start_with_reserved_raw_payload(serialized_request, build_start) .await @@ -1237,7 +1253,7 @@ impl Oplog for MultiLayerOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { self.primary .add_start_with_indexed_reserved_raw_payload(build_request) .await diff --git a/golem-worker-executor/src/services/oplog/plugin.rs b/golem-worker-executor/src/services/oplog/plugin.rs index 58cb5f7114..c024d05e77 100644 --- a/golem-worker-executor/src/services/oplog/plugin.rs +++ b/golem-worker-executor/src/services/oplog/plugin.rs @@ -17,7 +17,8 @@ use crate::model::event::InternalWorkerEvent; use crate::services::component::ComponentService; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, OpenOplogs, Oplog, - OplogAddReceipt, OplogConstructor, OplogService, OrderedOplogStart, ReservedRawStartBuilder, + OplogAddReceipt, OplogConstructor, OplogError, OplogFence, OplogService, OrderedOplogStart, + ReservedRawStartBuilder, }; use crate::services::shard::ShardService; use crate::services::worker_activator::WorkerActivator; @@ -30,6 +31,7 @@ use crate::workerctx::WorkerCtx; use anyhow::anyhow; use async_lock::{RwLock, RwLockUpgradableReadGuard}; use async_trait::async_trait; +use golem_common::model::ShardEpoch; use golem_common::model::account::AccountId; use golem_common::model::agent::{AgentMode, ParsedAgentId, Principal}; use golem_common::model::component::{ComponentId, ComponentRevision, InstalledPlugin}; @@ -518,6 +520,7 @@ struct CreateOplogConstructor { execution_status: read_only_lock::std::ReadOnlyLock, plugin_max_commit_count: usize, plugin_max_elapsed_time: Duration, + shard_epoch: Option, } impl CreateOplogConstructor { @@ -536,6 +539,7 @@ impl CreateOplogConstructor { execution_status: read_only_lock::std::ReadOnlyLock, plugin_max_commit_count: usize, plugin_max_elapsed_time: Duration, + shard_epoch: Option, ) -> Self { Self { owned_agent_id, @@ -551,6 +555,7 @@ impl CreateOplogConstructor { execution_status, plugin_max_commit_count, plugin_max_elapsed_time, + shard_epoch, } } } @@ -576,6 +581,7 @@ impl OplogConstructor for CreateOplogConstructor { self.initial_worker_metadata.clone(), self.last_known_status.clone(), self.execution_status.clone(), + self.shard_epoch, ) .await } else { @@ -587,6 +593,7 @@ impl OplogConstructor for CreateOplogConstructor { self.initial_worker_metadata.clone(), self.last_known_status.clone(), self.execution_status.clone(), + self.shard_epoch, ) .await } @@ -599,6 +606,7 @@ impl OplogConstructor for CreateOplogConstructor { self.initial_worker_metadata.clone(), self.last_known_status.clone(), self.execution_status.clone(), + self.shard_epoch, ) .await }; @@ -673,6 +681,7 @@ impl OplogService for ForwardingOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( @@ -691,6 +700,7 @@ impl OplogService for ForwardingOplogService { execution_status, self.plugin_max_commit_count, self.plugin_max_elapsed_time, + shard_epoch, ), ) .await @@ -704,6 +714,7 @@ impl OplogService for ForwardingOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( @@ -722,6 +733,7 @@ impl OplogService for ForwardingOplogService { execution_status, self.plugin_max_commit_count, self.plugin_max_elapsed_time, + shard_epoch, ), ) .await @@ -735,6 +747,7 @@ impl OplogService for ForwardingOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { self.oplogs .get_or_open( @@ -753,6 +766,7 @@ impl OplogService for ForwardingOplogService { execution_status, self.plugin_max_commit_count, self.plugin_max_elapsed_time, + shard_epoch, ), ) .await @@ -862,29 +876,29 @@ pub struct ForwardingOplog { enum ForwardingJob { Add { entry: OplogEntry, - done: tokio::sync::oneshot::Sender, + done: tokio::sync::oneshot::Sender>, }, AddDurableStreamBatch { make_batch: DurableStreamBatchBuilder, - done: tokio::sync::oneshot::Sender, String>>, + done: tokio::sync::oneshot::Sender, OplogError>>, }, AddPair { start: OplogEntry, make_second: Box OplogEntry + Send>, - done: tokio::sync::oneshot::Sender<(OplogIndex, OplogIndex)>, + done: tokio::sync::oneshot::Sender>, }, AddStart { serialized_request: Vec, build_start: ReservedRawStartBuilder, - done: tokio::sync::oneshot::Sender>, + done: tokio::sync::oneshot::Sender>, }, AddIndexedStart { build_request: IndexedReservedStartBuilder, - done: tokio::sync::oneshot::Sender>, + done: tokio::sync::oneshot::Sender>, }, Commit { level: CommitLevel, - done: tokio::sync::oneshot::Sender>, + done: tokio::sync::oneshot::Sender, OplogError>>, }, SetWorkerEventService { service: Arc, @@ -979,13 +993,17 @@ impl ForwardingOplog { ForwardingJob::Add { entry, done } => { let cache_entries = state.cache_is_required(); let cached_entry = cache_entries.then(|| entry.clone()); - let idx = state.inner.add(entry).await; - if let Some(entry) = cached_entry { - state.record_cached([(idx, entry)]); - } else { - state.record_uncached([idx]); + let result = state.inner.add(entry).await; + // A refused write is recorded nowhere: the inner oplog appended nothing, + // so the index it would have taken has to stay free. + if let Ok(idx) = &result { + if let Some(entry) = cached_entry { + state.record_cached([(*idx, entry)]); + } else { + state.record_uncached([*idx]); + } } - let _ = done.send(idx); + let _ = done.send(result); } ForwardingJob::AddDurableStreamBatch { make_batch, done } => { let cache_entries = state.cache_is_required(); @@ -1012,10 +1030,12 @@ impl ForwardingOplog { let cache_entries = state.cache_is_required(); let cached_entries = cache_entries.then(|| (start.clone(), second.clone())); let result = state.inner.add_pair(start, Box::new(move |_| second)).await; - if let Some((start, second)) = cached_entries { - state.record_cached([(result.0, start), (result.1, second)]); - } else { - state.record_uncached([result.0, result.1]); + if let Ok((start_idx, second_idx)) = &result { + if let Some((start, second)) = cached_entries { + state.record_cached([(*start_idx, start), (*second_idx, second)]); + } else { + state.record_uncached([*start_idx, *second_idx]); + } } let _ = done.send(result); } @@ -1060,40 +1080,46 @@ impl ForwardingOplog { let _ = done.send(result); } ForwardingJob::Commit { level, done } => { - let mut result = state.inner.commit(level).await; - // Update last_committed_idx from committed entries - if let Some(max_idx) = result.keys().max() - && *max_idx > state.last_committed_idx - { - state.last_committed_idx = *max_idx; - } - state.commit_count += 1; - if state.commit_count >= max_commit_count { - // Spanned inside the threshold check, not around the commit: - // this arm runs per oplog commit, the flush only every - // `max_commit_count` of them. The actor has no ambient span, - // so without this the flush would be untraceable. - // - // Named apart from the periodic `oplog_forwarding_flush` so the - // two triggers stay distinguishable in a trace backend. The link - // points at the worker's startup rather than at the commit that - // tripped the threshold: the actor receives commits over a - // channel, so the committing invocation's context is not - // available here. - state - .try_flush() - .instrument(related_span!( - flush_origin, - tracing::Level::INFO, - "oplog_forwarding_threshold_flush", - agent_id = %agent_id - )) - .await; + match state.inner.commit(level).await { + Err(error) => { + let _ = done.send(Err(error)); + } + Ok(mut result) => { + // Update last_committed_idx from committed entries + if let Some(max_idx) = result.keys().max() + && *max_idx > state.last_committed_idx + { + state.last_committed_idx = *max_idx; + } + state.commit_count += 1; + if state.commit_count >= max_commit_count { + // Spanned inside the threshold check, not around the commit: + // this arm runs per oplog commit, the flush only every + // `max_commit_count` of them. The actor has no ambient span, + // so without this the flush would be untraceable. + // + // Named apart from the periodic `oplog_forwarding_flush` so the + // two triggers stay distinguishable in a trace backend. The link + // points at the worker's startup rather than at the commit that + // tripped the threshold: the actor receives commits over a + // channel, so the committing invocation's context is not + // available here. + state + .try_flush() + .instrument(related_span!( + flush_origin, + tracing::Level::INFO, + "oplog_forwarding_threshold_flush", + agent_id = %agent_id + )) + .await; + } + // Merge entries committed directly to inner during flush + // so the Worker folds them into AgentStatusRecord + result.append(&mut state.pending_direct_commits); + let _ = done.send(Ok(result)); + } } - // Merge entries committed directly to inner during flush - // so the Worker folds them into AgentStatusRecord - result.append(&mut state.pending_direct_commits); - let _ = done.send(result); } ForwardingJob::SetWorkerEventService { service, done } => { state.worker_event_service = Some(service); @@ -1229,7 +1255,7 @@ impl Oplog for ForwardingOplog { async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { self.run_job(|done| ForwardingJob::AddDurableStreamBatch { make_batch, done }) .await } @@ -1238,7 +1264,10 @@ impl Oplog for ForwardingOplog { self.inner.drop_prefix(last_dropped_id).await } - async fn commit(&self, level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + level: CommitLevel, + ) -> Result, OplogError> { self.run_job(|done| ForwardingJob::Commit { level, done }) .await } @@ -1304,7 +1333,7 @@ impl Oplog for ForwardingOplog { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), OplogError> { self.run_job(|done| ForwardingJob::AddPair { start, make_second, @@ -1317,7 +1346,7 @@ impl Oplog for ForwardingOplog { &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result { + ) -> Result { self.run_job(|done| ForwardingJob::AddStart { serialized_request, build_start, @@ -1329,7 +1358,7 @@ impl Oplog for ForwardingOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { self.run_job(|done| ForwardingJob::AddIndexedStart { build_request, done, @@ -1578,14 +1607,19 @@ impl ForwardingOplogState { target_agent = %id, "Oplog processor: resolved target plugin worker" ); - self.write_checkpoint( - grant_id, - &id, - live.confirmed_up_to, - live.confirmed_up_to, - live.last_batch_start, - ) - .await; + if self + .write_checkpoint( + grant_id, + &id, + live.confirmed_up_to, + live.confirmed_up_to, + live.last_batch_start, + ) + .await + .is_err() + { + return; + } if let Some(s) = self.plugin_state.get_mut(&grant_id) { s.target_agent_id = Some(id.clone()); } @@ -1628,14 +1662,19 @@ impl ForwardingOplogState { } if !is_retry { - self.write_checkpoint( - grant_id, - &target_agent_id, - live.confirmed_up_to, - batch_end, - batch_start, - ) - .await; + if self + .write_checkpoint( + grant_id, + &target_agent_id, + live.confirmed_up_to, + batch_end, + batch_start, + ) + .await + .is_err() + { + return; + } if let Some(s) = self.plugin_state.get_mut(&grant_id) { s.sending_up_to = batch_end; } @@ -1671,14 +1710,19 @@ impl ForwardingOplogState { "Oplog processor: batch enqueued successfully" ); // Enqueue succeeded — immediately confirm - self.write_checkpoint( - grant_id, - &target_agent_id, - batch_end, - batch_end, - batch_start, - ) - .await; + if self + .write_checkpoint( + grant_id, + &target_agent_id, + batch_end, + batch_end, + batch_start, + ) + .await + .is_err() + { + return; + } if let Some(s) = self.plugin_state.get_mut(&grant_id) { s.confirmed_up_to = batch_end; s.sending_up_to = batch_end; @@ -1829,6 +1873,13 @@ impl ForwardingOplogState { } /// Write an OplogProcessorCheckpoint entry, commit it, and update index tracking. + /// + /// `Err` means the agent's oplog was fenced: its shard has a new owner, so nothing more may be + /// written and forwarding stops. Giving the agent up is deliberately not this layer's call - a + /// storage decorator has no business stopping agents - and it does not need to be: the fence + /// latches on the oplog, so the worker's own commit path is refused too and relinquishes it. + /// + /// Any other storage failure keeps the fail-stop behaviour it has always had. async fn write_checkpoint( &mut self, grant_id: EnvironmentPluginGrantId, @@ -1836,7 +1887,7 @@ impl ForwardingOplogState { confirmed_up_to: OplogIndex, sending_up_to: OplogIndex, last_batch_start: OplogIndex, - ) { + ) -> Result<(), OplogFence> { let checkpoint = OplogEntry::OplogProcessorCheckpoint { timestamp: golem_common::model::Timestamp::now_utc(), plugin_grant_id: grant_id, @@ -1847,19 +1898,39 @@ impl ForwardingOplogState { }; let cache_entries = self.cache_is_required(); let cached_checkpoint = cache_entries.then(|| checkpoint.clone()); - let idx = self.inner.add(checkpoint).await; + let idx = match self.inner.add(checkpoint).await { + Ok(idx) => idx, + Err(OplogError::Fenced(fence)) => return Err(self.stop_forwarding(fence)), + Err(error) => panic!("oplog write: {error}"), + }; if let Some(checkpoint) = cached_checkpoint { self.record_cached([(idx, checkpoint)]); } else { self.record_uncached([idx]); } - let committed = self.inner.commit(CommitLevel::Always).await; + let committed = match self.inner.commit(CommitLevel::Always).await { + Ok(committed) => committed, + Err(OplogError::Fenced(fence)) => return Err(self.stop_forwarding(fence)), + Err(error) => panic!("oplog write: {error}"), + }; if let Some(max_idx) = committed.keys().max().copied() { self.last_committed_idx = self.last_committed_idx.max(max_idx); } // Track all directly committed entries so ForwardingOplog::commit() // can surface them to the Worker for status folding self.pending_direct_commits.extend(committed); + Ok(()) + } + + /// Logs a fenced checkpoint once and hands the fence back to the caller, which stops + /// forwarding for this agent. + fn stop_forwarding(&self, fence: OplogFence) -> OplogFence { + tracing::info!( + source_agent = %self.initial_worker_metadata.agent_id, + expected_epoch = fence.expected_epoch.0, + "Oplog processor: checkpoint fenced, the shard has a new owner - forwarding stopped" + ); + fence } /// Prune buffer: drain entries that ALL active/in-flight plugins have confirmed past. @@ -2070,14 +2141,19 @@ impl ForwardingOplogState { } } - self.write_checkpoint( - grant_id, - &new_target, - confirmed, - confirmed, - last_batch_start, - ) - .await; + if self + .write_checkpoint( + grant_id, + &new_target, + confirmed, + confirmed, + last_batch_start, + ) + .await + .is_err() + { + return; + } if let Some(s) = self.plugin_state.get_mut(&grant_id) { s.target_agent_id = Some(new_target.clone()); } @@ -2530,14 +2606,14 @@ mod tests { *idx = idx.next(); entries.push(entry); let result = *idx; - Box::pin(async move { result }) + Box::pin(async move { Ok(result) }) } async fn add_pair( &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), crate::services::oplog::OplogError> { let mut entries = self.entries.lock().unwrap(); let mut idx = self.current_idx.lock().unwrap(); *idx = idx.next(); @@ -2546,13 +2622,13 @@ mod tests { *idx = idx.next(); let second_idx = *idx; entries.push(make_second(first_idx)); - (first_idx, second_idx) + Ok((first_idx, second_idx)) } async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { let mut entries = self.entries.lock().unwrap(); let mut idx = self.current_idx.lock().unwrap(); let records = make_batch(idx.next()); @@ -2570,9 +2646,9 @@ mod tests { &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result { + ) -> Result { let entry = build_start(RawOplogPayload::SerializedInline(serialized_request))?; - let index = self.add(entry.clone()).await; + let index = self.add(entry.clone()).await?; Ok(OrderedOplogStart { index, entry, @@ -2583,7 +2659,7 @@ mod tests { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let mut entries = self.entries.lock().unwrap(); let mut idx = self.current_idx.lock().unwrap(); let index = idx.next(); @@ -2602,7 +2678,10 @@ mod tests { 0 } - async fn commit(&self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + _level: CommitLevel, + ) -> Result, crate::services::oplog::OplogError> { let entries = self.entries.lock().unwrap(); let current = *self.current_idx.lock().unwrap(); let mut committed = self.committed_idx.lock().unwrap(); @@ -2613,7 +2692,7 @@ mod tests { result.insert(idx, entries[(idx.as_u64() - 1) as usize].clone()); } *committed = current; - result + Ok(result) } async fn current_oplog_index(&self) -> OplogIndex { @@ -2840,8 +2919,8 @@ mod tests { timestamp: Timestamp::now_utc(), delta: 200, }; - inner.add(entry1.clone()).await; - inner.add(entry2.clone()).await; + inner.add(entry1.clone()).await.unwrap(); + inner.add(entry2.clone()).await.unwrap(); let mut state = ForwardingOplogState { buffer: Some(VecDeque::from([entry1, entry2])), @@ -2892,7 +2971,7 @@ mod tests { timestamp: Timestamp::now_utc(), delta: 100, }; - inner.add(entry.clone()).await; + inner.add(entry.clone()).await.unwrap(); let mut state = ForwardingOplogState { buffer: Some(VecDeque::from([entry])), @@ -2968,9 +3047,10 @@ mod tests { timestamp: Timestamp::now_utc(), entity_parent_start_index: None, }) - .await; + .await + .unwrap(); - assert_eq!(first.await, OplogIndex::INITIAL); + assert_eq!(first.await.unwrap(), OplogIndex::INITIAL); assert_eq!(second, OplogIndex::INITIAL.next()); } @@ -3041,7 +3121,7 @@ mod tests { second_pending.wait().await.unwrap(); // With max_commit_count = 1 the first commit triggers a flush to the plugin. - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let sends = recording_plugin.sends().await; assert_eq!(sends.len(), 1, "Expected exactly one batch"); @@ -3128,10 +3208,11 @@ mod tests { let (oplog, inner, recording_plugin, status_writer) = test_forwarding_oplog(HashSet::new(), grant_id, 1).await; - oplog.add(grow_memory(1)).await; + oplog.add(grow_memory(1)).await.unwrap(); oplog .add_pair(grow_memory(2), Box::new(|_| grow_memory(3))) - .await; + .await + .unwrap(); oplog .add_start_with_reserved_raw_payload(Vec::new(), Box::new(|_| Ok(grow_memory(4)))) .await @@ -3166,13 +3247,13 @@ mod tests { assert_eq!(uncached.buffer_start_idx, uncached.last_oplog_idx.next()); publish_status(&status_writer, HashSet::from([grant_id]), None); - oplog.add(grow_memory(8)).await; + oplog.add(grow_memory(8)).await.unwrap(); let cached = oplog.inspect_state().await; assert_eq!(cached.buffer_len, Some(1)); assert_eq!(cached.buffer_start_idx, OplogIndex::from_u64(8)); assert_eq!(cached.last_oplog_idx, OplogIndex::from_u64(8)); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let sends = recording_plugin.sends().await; assert_eq!(sends.len(), 1); assert_eq!(sends[0].initial_oplog_index, OplogIndex::INITIAL); @@ -3196,7 +3277,7 @@ mod tests { test_forwarding_oplog(HashSet::new(), grant_id, usize::MAX).await; let started = std::time::Instant::now(); for delta in 0..APPENDS { - uncached.add(grow_memory(delta)).await; + uncached.add(grow_memory(delta)).await.unwrap(); } let uncached_elapsed = started.elapsed(); assert_eq!(uncached.inspect_state().await.buffer_len, None); @@ -3205,7 +3286,7 @@ mod tests { test_forwarding_oplog(HashSet::from([grant_id]), grant_id, usize::MAX).await; let started = std::time::Instant::now(); for delta in 0..APPENDS { - cached.add(grow_memory(delta)).await; + cached.add(grow_memory(delta)).await.unwrap(); } let cached_elapsed = started.elapsed(); assert_eq!( @@ -3225,8 +3306,8 @@ mod tests { let grant_id = EnvironmentPluginGrantId::new(); let (oplog, inner, recording_plugin, status_writer) = test_forwarding_oplog(HashSet::new(), grant_id, 1).await; - oplog.add(grow_memory(1)).await; - oplog.add(grow_memory(2)).await; + oplog.add(grow_memory(1)).await.unwrap(); + oplog.add(grow_memory(2)).await.unwrap(); let target = recording_plugin.target_agent_id.clone(); publish_status( @@ -3242,8 +3323,8 @@ mod tests { }, )), ); - oplog.add(grow_memory(3)).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(grow_memory(3)).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); let sends = recording_plugin.sends().await; assert_eq!(sends.len(), 1); @@ -3257,19 +3338,19 @@ mod tests { let grant_id = EnvironmentPluginGrantId::new(); let (oplog, inner, recording_plugin, status_writer) = test_forwarding_oplog(HashSet::from([grant_id]), grant_id, 1).await; - oplog.add(grow_memory(1)).await; + oplog.add(grow_memory(1)).await.unwrap(); publish_status(&status_writer, HashSet::new(), None); - oplog.add(grow_memory(2)).await; + oplog.add(grow_memory(2)).await.unwrap(); let skipped = oplog.inspect_state().await; assert_eq!(skipped.buffer_len, None); assert_eq!(skipped.buffer_start_idx, skipped.last_oplog_idx.next()); publish_status(&status_writer, HashSet::from([grant_id]), None); - oplog.add(grow_memory(3)).await; + oplog.add(grow_memory(3)).await.unwrap(); let resumed = oplog.inspect_state().await; assert_eq!(resumed.buffer_len, Some(1)); assert_eq!(resumed.buffer_start_idx, OplogIndex::from_u64(3)); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let sends = recording_plugin.sends().await; assert_eq!(sends.len(), 1); @@ -3284,7 +3365,7 @@ mod tests { let grant_id = EnvironmentPluginGrantId::new(); let (oplog, _inner, _recording_plugin, status_writer) = test_forwarding_oplog(HashSet::from([grant_id]), grant_id, usize::MAX).await; - oplog.add(grow_memory(1)).await; + oplog.add(grow_memory(1)).await.unwrap(); assert_eq!(oplog.inspect_state().await.buffer_len, Some(1)); publish_status(&status_writer, HashSet::new(), None); @@ -3337,13 +3418,13 @@ mod tests { let (oplog, _inner, recording_plugin, status_writer) = test_forwarding_oplog(HashSet::from([grant_id]), grant_id, 1).await; recording_plugin.fail_next_send_from_remote_target(); - oplog.add(grow_memory(1)).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(grow_memory(1)).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); publish_status(&status_writer, HashSet::new(), None); - oplog.add(grow_memory(2)).await; + oplog.add(grow_memory(2)).await.unwrap(); assert!(oplog.inspect_state().await.buffer_len.is_some()); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let sends = recording_plugin.sends().await; assert_eq!(sends.len(), 2); @@ -3368,8 +3449,8 @@ mod tests { let grant_id = EnvironmentPluginGrantId::new(); let (oplog, _inner, _recording_plugin, _status_writer) = test_forwarding_oplog(HashSet::from([grant_id]), grant_id, usize::MAX).await; - oplog.add(grow_memory(1)).await; - let initial_commit = oplog.commit(CommitLevel::Always).await; + oplog.add(grow_memory(1)).await.unwrap(); + let initial_commit = oplog.commit(CommitLevel::Always).await.unwrap(); assert_eq!(initial_commit.len(), 1); oplog.tick().await; @@ -3378,7 +3459,7 @@ mod tests { assert_eq!(pruned.buffer_len, None); assert_eq!(pruned.buffer_start_idx, pruned.last_oplog_idx.next()); - let checkpoint_commit = oplog.commit(CommitLevel::Always).await; + let checkpoint_commit = oplog.commit(CommitLevel::Always).await.unwrap(); assert_eq!( checkpoint_commit .values() @@ -3386,6 +3467,6 @@ mod tests { .count(), 3 ); - assert!(oplog.commit(CommitLevel::Always).await.is_empty()); + assert!(oplog.commit(CommitLevel::Always).await.unwrap().is_empty()); } } diff --git a/golem-worker-executor/src/services/oplog/primary.rs b/golem-worker-executor/src/services/oplog/primary.rs index 66bef377d5..bbd963c4e6 100644 --- a/golem-worker-executor/src/services/oplog/primary.rs +++ b/golem-worker-executor/src/services/oplog/primary.rs @@ -23,8 +23,9 @@ use crate::services::oplog::reader::{ }; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, OpenOplogs, Oplog, - OplogAddReceipt, OplogConstructor, OplogService, OrderedOplogStart, PendingUpload, - ReservedPayload, ReservedRawStartBuilder, cursor_value, next_scan_cursor, scan_modes, + OplogAddReceipt, OplogConstructor, OplogError, OplogFence, OplogService, OrderedOplogStart, + PendingUpload, ReservedPayload, ReservedRawStartBuilder, cursor_value, next_scan_cursor, + scan_modes, }; use crate::storage::indexed::{ IndexedStorage, IndexedStorageError, IndexedStorageLabelledApi, IndexedStorageMetaNamespace, @@ -34,6 +35,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures::FutureExt; use golem_common::model::RetryConfig; +use golem_common::model::ShardEpoch; use golem_common::model::account::AccountId; use golem_common::model::agent::AgentMode; use golem_common::model::component::ComponentId; @@ -55,12 +57,39 @@ use std::sync::Arc; use std::time::Duration; use tracing::{error, warn}; +/// Runs a storage operation under the retry policy, panicking on anything it cannot retry away. +/// +/// Reads, deletions and prefix drops keep this shape: a permanent failure there is a broken +/// deployment, and failing fast is the long-standing contract. async fn retry_storage_op( retry_config: &RetryConfig, op_name: &str, key: &str, - mut op: F, + op: F, ) -> T +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + match retry_storage_op_fenceable(retry_config, op_name, key, op).await { + Ok(val) => val, + Err(err) => panic!("Indexed storage operation '{op_name}' failed for key '{key}': {err}"), + } +} + +/// As [`retry_storage_op`], but hands a fence back instead of panicking on it. +/// +/// A fenced write is not a storage failure: the storage is healthy and refused the write on +/// purpose, because this executor no longer owns the agent's shard. Retrying cannot change that, +/// and panicking would take the whole executor down over one agent that simply moved. Every other +/// permanent failure still panics, so the fail-stop contract is unchanged for everything else - +/// including the primary-key collision that has always been the crude fence. +async fn retry_storage_op_fenceable( + retry_config: &RetryConfig, + op_name: &str, + key: &str, + mut op: F, +) -> Result where F: FnMut() -> Fut, Fut: std::future::Future>, @@ -69,7 +98,8 @@ where loop { attempts += 1; match op().await { - Ok(val) => return val, + Ok(val) => return Ok(val), + Err(err @ IndexedStorageError::Fenced { .. }) => return Err(err), Err(IndexedStorageError::Transient(msg)) => { if let Some(delay) = get_delay(retry_config, attempts) { record_oplog_storage_retry(op_name); @@ -151,17 +181,18 @@ impl SerializedOplogAppend { namespace: &IndexedStorageNamespace, api_name: &'static str, key: &str, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { let storage = indexed_storage.with_entity("oplog", api_name, "entry"); match self { Self::Entry((id, value)) => { storage - .append_raw(namespace.clone(), key, *id, value.to_vec()) + .append_raw(namespace.clone(), key, *id, value.to_vec(), shard_epoch) .await } Self::Batch(entries) => { storage - .append_many_raw(namespace, key, entries.clone()) + .append_many_raw(namespace, key, entries.clone(), shard_epoch) .await } } @@ -176,19 +207,27 @@ async fn retry_oplog_append( api_name: &'static str, key: &str, append: SerializedOplogAppend, -) { + shard_epoch: Option, +) -> Result<(), IndexedStorageError> { let mut attempts = 0u32; let mut write_may_have_committed = false; loop { attempts += 1; let error = match append - .write(indexed_storage, namespace, api_name, key) + .write(indexed_storage, namespace, api_name, key, shard_epoch) .await { - Ok(()) => return, + Ok(()) => return Ok(()), Err(error) => error, }; + // The storage refused the write because the shard has a new owner. Not a transient + // failure to retry and not an indeterminate one to reconcile: it is a deliberate refusal, + // so hand it back and let the caller give up this one agent instead of aborting. + if matches!(error, IndexedStorageError::Fenced { .. }) { + return Err(error); + } + let retryable = match &error { IndexedStorageError::Indeterminate(_) => { write_may_have_committed = true; @@ -209,6 +248,8 @@ async fn retry_oplog_append( } false } + // Returned above; named here only because the guard does not make this exhaustive. + IndexedStorageError::Fenced { .. } => unreachable!("a fence returns before this match"), }; if write_may_have_committed { @@ -221,7 +262,7 @@ async fn retry_oplog_append( ) .await { - Some(true) => return, + Some(true) => return Ok(()), Some(false) => panic!( "Indexed storage operation '{op_name}' failed for key '{key}' and the indeterminate write did not match storage: {error}" ), @@ -255,6 +296,48 @@ async fn retry_oplog_append( } } +/// Records the epoch this executor is allowed to write `key` with, and reports the fence when +/// the stored record is already ahead of it. +/// +/// Monotonic on the storage side, so a re-grant at a higher epoch takes the oplog over while an +/// executor holding a stale one cannot claim it back. Written before the oplog's first entry - +/// an absent record fences too, which is what closes the window between creating an oplog and +/// recording who owns it. +async fn record_owning_epoch( + indexed_storage: &(dyn IndexedStorage + Send + Sync), + retry_config: &RetryConfig, + owned_agent_id: &OwnedAgentId, + agent_mode: AgentMode, + key: &str, + shard_epoch: ShardEpoch, +) -> Option { + let outcome = retry_storage_op_fenceable(retry_config, "upsert_oplog_metadata", key, || { + let ns = IndexedStorageNamespace::OpLog { + agent_id: owned_agent_id.agent_id(), + agent_mode, + }; + async move { + indexed_storage + .upsert_oplog_metadata("oplog", "upsert_oplog_metadata", ns, key, shard_epoch) + .await + } + }) + .await; + + match outcome { + Ok(()) => None, + Err(IndexedStorageError::Fenced { + expected, actual, .. + }) => Some(OplogFence { + agent_id: owned_agent_id.agent_id(), + expected_epoch: expected, + actual_epoch: actual, + }), + // `retry_storage_op_fenceable` panics on every other permanent failure. + Err(other) => unreachable!("unexpected storage error: {other}"), + } +} + async fn read_persisted_oplog_entries( indexed_storage: Arc, namespace: IndexedStorageNamespace, @@ -339,6 +422,7 @@ impl PrimaryOplogService { op_name: &str, api_name: &'static str, entry: &OplogEntry, + shard_epoch: Option, ) { let key = Self::oplog_key(&owned_agent_id.agent_id); let namespace = IndexedStorageNamespace::OpLog { @@ -358,8 +442,19 @@ impl PrimaryOplogService { api_name, &key, SerializedOplogAppend::Entry((1, value)), + shard_epoch, ) - .await; + .await + .unwrap_or_else(|err| { + // Only a fence reaches here - every other permanent failure already panicked inside + // `retry_oplog_append`. Nothing more is written: `open` records the epoch again and, + // being refused there too, hands back an oplog that refuses every write. + warn!( + agent_id = %owned_agent_id, + error = %err, + "Initial oplog entry fenced: the shard has a new owner" + ); + }); } async fn get_last_index_from_storage( @@ -482,6 +577,7 @@ impl OplogService for PrimaryOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { record_oplog_call("create"); @@ -506,14 +602,34 @@ impl OplogService for PrimaryOplogService { panic!("oplog for worker {owned_agent_id} already exists in indexed storage") } - self.append_initial_entry( - owned_agent_id, - agent_mode, - "create_append", - "create", - &initial_entry, - ) - .await; + // The record goes in before the first entry. If it is refused, this executor has already + // lost the shard: skip the append entirely and let `open` below hand back an oplog that + // refuses every write. + let fenced_at_create = match shard_epoch { + Some(epoch) => record_owning_epoch( + &*self.indexed_storage, + &self.retry_config, + owned_agent_id, + agent_mode, + &key, + epoch, + ) + .await + .is_some(), + None => false, + }; + + if !fenced_at_create { + self.append_initial_entry( + owned_agent_id, + agent_mode, + "create_append", + "create", + &initial_entry, + shard_epoch, + ) + .await; + } self.open( owned_agent_id, @@ -522,6 +638,7 @@ impl OplogService for PrimaryOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ) .await } @@ -534,20 +651,40 @@ impl OplogService for PrimaryOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { record_oplog_call("create_fresh"); // The caller guarantees the agent id is freshly derived and unused, so // the existence probe performed by `create` is skipped: the initial - // entry is appended directly without any prior read. - self.append_initial_entry( - owned_agent_id, - agent_mode, - "create_fresh_append", - "create_fresh", - &initial_entry, - ) - .await; + // entry is appended directly without any prior read. The epoch record still goes in + // first - a fresh agent id does not mean a fresh shard. + let key = Self::oplog_key(&owned_agent_id.agent_id); + let fenced_at_create = match shard_epoch { + Some(epoch) => record_owning_epoch( + &*self.indexed_storage, + &self.retry_config, + owned_agent_id, + agent_mode, + &key, + epoch, + ) + .await + .is_some(), + None => false, + }; + + if !fenced_at_create { + self.append_initial_entry( + owned_agent_id, + agent_mode, + "create_fresh_append", + "create_fresh", + &initial_entry, + shard_epoch, + ) + .await; + } self.open( owned_agent_id, @@ -556,6 +693,7 @@ impl OplogService for PrimaryOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ) .await } @@ -568,6 +706,7 @@ impl OplogService for PrimaryOplogService { initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { record_oplog_call("open"); @@ -581,6 +720,7 @@ impl OplogService for PrimaryOplogService { .get_or_open( &owned_agent_id.agent_id, CreateOplogConstructor::new( + shard_epoch, self.indexed_storage.clone(), self.blob_storage.clone(), self.replicas, @@ -620,6 +760,22 @@ impl OplogService for PrimaryOplogService { let is = self.indexed_storage.clone(); let agent_id = owned_agent_id.agent_id(); let key = Self::oplog_key(&owned_agent_id.agent_id); + // The epoch record goes before the entries: a writer still holding this oplog open is + // then refused by the absent record, instead of appending entries back into an oplog + // that is being removed. + retry_storage_op(&self.retry_config, "delete_oplog_metadata", &key, || { + let is = is.clone(); + let ns = IndexedStorageNamespace::OpLog { + agent_id: agent_id.clone(), + agent_mode, + }; + let key = key.clone(); + async move { + is.delete_oplog_metadata("oplog", "delete_oplog_metadata", ns, &key) + .await + } + }) + .await; retry_storage_op(&self.retry_config, "delete", &key, || { let is = is.clone(); let ns = IndexedStorageNamespace::OpLog { @@ -796,11 +952,13 @@ struct CreateOplogConstructor { agent_mode: AgentMode, account_id: AccountId, stream_session_index: Option>, + shard_epoch: Option, } impl CreateOplogConstructor { #[allow(clippy::too_many_arguments)] fn new( + shard_epoch: Option, indexed_storage: Arc, blob_storage: Arc, replicas: u8, @@ -815,6 +973,7 @@ impl CreateOplogConstructor { stream_session_index: Option>, ) -> Self { Self { + shard_epoch, indexed_storage, blob_storage, replicas, @@ -846,7 +1005,26 @@ impl OplogConstructor for CreateOplogConstructor { .await } }; + // Recorded before the oplog is usable, so an executor whose shard has moved is refused + // at its very first write rather than after replaying the new owner's entries. + let fence = match self.shard_epoch { + Some(shard_epoch) => { + record_owning_epoch( + &*self.indexed_storage, + &self.retry_config, + &self.owned_agent_id, + self.agent_mode, + &self.key, + shard_epoch, + ) + .await + } + None => None, + }; + Arc::new(PrimaryOplog::new( + self.shard_epoch, + fence, self.indexed_storage, self.blob_storage, self.replicas, @@ -901,6 +1079,11 @@ struct PrimaryOplog { key: String, owned_agent_id: OwnedAgentId, agent_mode: AgentMode, + /// The epoch the actor's state asserts on every append, copied here so that reading it does + /// not have to go through the actor. Fixed for the oplog's lifetime. + shard_epoch: Option, + /// The refusal the actor's state has latched, shared so the handle can report it. + fence: Arc>, stream_session_index: Option>, close: Option>, } @@ -912,29 +1095,29 @@ struct PrimaryOplog { enum OplogJob { Add { entry: OplogEntry, - done: tokio::sync::oneshot::Sender, + done: tokio::sync::oneshot::Sender>, }, AddDurableStreamBatch { make_batch: DurableStreamBatchBuilder, - done: tokio::sync::oneshot::Sender, String>>, + done: tokio::sync::oneshot::Sender, OplogError>>, }, AddPair { start: OplogEntry, make_second: Box OplogEntry + Send>, - done: tokio::sync::oneshot::Sender<(OplogIndex, OplogIndex)>, + done: tokio::sync::oneshot::Sender>, }, AddStart { serialized_request: Vec, build_start: ReservedRawStartBuilder, - done: tokio::sync::oneshot::Sender>, + done: tokio::sync::oneshot::Sender>, }, AddIndexedStart { build_request: IndexedReservedStartBuilder, - done: tokio::sync::oneshot::Sender>, + done: tokio::sync::oneshot::Sender>, }, Commit { level: CommitLevel, - done: tokio::sync::oneshot::Sender>, + done: tokio::sync::oneshot::Sender, OplogError>>, }, Flush { done: tokio::sync::oneshot::Sender<()>, @@ -1001,6 +1184,8 @@ impl Drop for PrimaryOplog { impl PrimaryOplog { #[allow(clippy::too_many_arguments)] fn new( + shard_epoch: Option, + fence: Option, indexed_storage: Arc, blob_storage: Arc, replicas: u8, @@ -1017,7 +1202,13 @@ impl PrimaryOplog { ) -> Self { let account_id_label = account_id.to_string(); let environment_id_label = owned_agent_id.environment_id().to_string(); + let fence = Arc::new(match fence { + Some(fence) => std::sync::OnceLock::from(fence), + None => std::sync::OnceLock::new(), + }); let mut state = PrimaryOplogState { + shard_epoch, + fence: fence.clone(), indexed_storage, blob_storage, replicas, @@ -1048,10 +1239,14 @@ impl PrimaryOplog { OplogJob::Add { entry, done } => { record_oplog_call("add"); let idx = state.push(entry); - if state.over_commit_threshold() { - state.commit(CommitLevel::Always).await; - } - let _ = done.send(idx); + // A threshold commit failing must fail the `add` that triggered it: the + // caller would otherwise be told its entry landed when the batch it was + // folded into was refused. + let result = match state.maybe_commit().await { + Ok(()) => Ok(idx), + Err(error) => Err(error), + }; + let _ = done.send(result); } OplogJob::AddDurableStreamBatch { make_batch, done } => { record_oplog_call("add_durable_stream_batch"); @@ -1079,9 +1274,11 @@ impl PrimaryOplog { } Ok(result) }); - if result.is_ok() && state.over_commit_threshold() { - state.commit(CommitLevel::Always).await; - } + let result = match (result, state.maybe_commit().await) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), _) => Err(error.into()), + (Ok(_), Err(error)) => Err(error), + }; let _ = done.send(result); } OplogJob::AddPair { @@ -1093,10 +1290,11 @@ impl PrimaryOplog { let first_idx = state.push(start); let second = make_second(first_idx); let second_idx = state.push(second); - if state.over_commit_threshold() { - state.commit(CommitLevel::Always).await; - } - let _ = done.send((first_idx, second_idx)); + // Both halves of the pair share the threshold commit, so a refused + // commit fails the pair rather than reporting a write that was rolled + // back. + let result = state.maybe_commit().await.map(|()| (first_idx, second_idx)); + let _ = done.send(result); } OplogJob::AddStart { serialized_request, @@ -1136,9 +1334,11 @@ impl PrimaryOplog { Err(err) => Err(err), } }; - if result.is_ok() && state.over_commit_threshold() { - state.commit(CommitLevel::Always).await; - } + let result = match (result, state.maybe_commit().await) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), _) => Err(error.into()), + (Ok(_), Err(error)) => Err(error), + }; let _ = done.send(result); } OplogJob::AddIndexedStart { @@ -1163,21 +1363,34 @@ impl PrimaryOplog { }) }, ); - if result.is_ok() && state.over_commit_threshold() { - state.commit(CommitLevel::Always).await; - } + let result = match (result, state.maybe_commit().await) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), _) => Err(error.into()), + (Ok(_), Err(error)) => Err(error), + }; let _ = done.send(result); } OplogJob::Commit { level, done } => { let previously_committed_through = state.last_committed_idx; - let committed = state.commit(level).await; - let result = state - .committed_since_last_report(previously_committed_through, committed) - .await; + let result = match state.commit(level).await { + Ok(committed) => Ok(state + .committed_since_last_report( + previously_committed_through, + committed, + ) + .await), + Err(error) => Err(error), + }; let _ = done.send(result); } OplogJob::Flush { done } => { - state.commit(CommitLevel::Always).await; + // The caller gets no error to act on. A fence is latched on the state, so + // every later write fails on it without asking the storage again; a + // transient storage failure is fatal here as everywhere else. + match state.commit(CommitLevel::Always).await { + Ok(_) | Err(OplogError::Fenced(_)) => {} + Err(error) => panic!("oplog write: {error}"), + } let _ = done.send(()); } OplogJob::DropPrefix { @@ -1271,6 +1484,8 @@ impl PrimaryOplog { key, owned_agent_id, agent_mode, + shard_epoch, + fence, stream_session_index, close: Some(close), } @@ -1465,6 +1680,18 @@ struct PrimaryOplogState { /// any buffered entries, so no committed entry can reference a not-yet-written blob. pending_uploads: Vec, durable_stream_sessions: super::raw_session::RawSessionCache, + /// The shard epoch this executor held for the agent's shard when the oplog was opened, and + /// the one every append asserts. + /// + /// Cached at open rather than read per write: one live oplog is one ownership generation, and + /// this is the value its metadata row was written with. A renewal never changes it - an epoch + /// only moves when the shard changes owner, and then this oplog is the losing side. + shard_epoch: Option, + /// Set once a write has been refused, or at open when the epoch record already belonged to a + /// newer owner. Every later write fails on it immediately: the oplog is another executor's + /// now, so there is nothing to be gained by asking the storage again. Shared with the handle, + /// which answers [`Oplog::fence`] from it without a round trip through the actor. + fence: Arc>, } impl PrimaryOplogState { @@ -1539,9 +1766,18 @@ impl PrimaryOplogState { } } - async fn append(&mut self, entries: Vec) -> BTreeMap { + async fn append( + &mut self, + entries: Vec, + ) -> Result, OplogError> { record_oplog_call("append"); + // Already refused once: fail fast rather than re-asking the storage for every entry the + // guest goes on to produce before it notices it has been given up. + if let Some(fence) = self.fence.get() { + return Err(OplogError::Fenced(fence.clone())); + } + // Commit barrier: every deferred external payload reserved during this session must be // durably written to blob storage before the entries (which may reference it) are persisted // to indexed storage. `append` flushes the whole buffer, so waiting on all outstanding @@ -1560,7 +1796,7 @@ impl PrimaryOplogState { } if entries.is_empty() { - return BTreeMap::new(); + return Ok(BTreeMap::new()); } let entry_count = entries.len() as u64; @@ -1592,8 +1828,17 @@ impl PrimaryOplogState { "append", &self.key, SerializedOplogAppend::Batch(serialized_pairs), + self.shard_epoch, ) - .await; + .await + .map_err(|err| Self::as_oplog_error(&self.owned_agent_id, err)) + .inspect_err(|err| { + if let OplogError::Fenced(fence) = err { + let _ = self.fence.set(fence.clone()); + // The drained entries are dropped: this oplog is not ours to write. + self.pending_uploads.clear(); + } + })?; record_storage_bytes_written( STORAGE_TYPE_OPLOG, @@ -1609,11 +1854,34 @@ impl PrimaryOplogState { ); self.last_committed_idx = last_idx; - BTreeMap::from_iter( + Ok(BTreeMap::from_iter( pairs .into_iter() .map(|(idx, entry)| (OplogIndex::from_u64(idx), entry)), - ) + )) + } + + /// Commits if the buffer is over the threshold. Separated out so the actor arms can fold a + /// threshold-commit failure into the job that triggered it. + async fn maybe_commit(&mut self) -> Result<(), OplogError> { + if self.over_commit_threshold() { + self.commit(CommitLevel::Always).await?; + } + Ok(()) + } + + /// Names the agent on a storage error, so the worker that hit it can be given up by id. + fn as_oplog_error(owned_agent_id: &OwnedAgentId, err: IndexedStorageError) -> OplogError { + match err { + IndexedStorageError::Fenced { + expected, actual, .. + } => OplogError::Fenced(OplogFence { + agent_id: owned_agent_id.agent_id(), + expected_epoch: expected, + actual_epoch: actual, + }), + other => OplogError::Storage(other.to_string()), + } } /// Pushes an entry into the in-memory buffer and advances the oplog index, @@ -1654,7 +1922,10 @@ impl PrimaryOplogState { self.buffer.len() > self.max_operations_before_commit as usize } - async fn commit(&mut self, _level: CommitLevel) -> BTreeMap { + async fn commit( + &mut self, + _level: CommitLevel, + ) -> Result, OplogError> { record_oplog_call("commit"); let entries = self.buffer.drain(..).collect::>(); @@ -1758,7 +2029,7 @@ impl Oplog for PrimaryOplog { async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { self.run_job(|done| OplogJob::AddDurableStreamBatch { make_batch, done }) .await } @@ -1767,7 +2038,7 @@ impl Oplog for PrimaryOplog { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), OplogError> { self.run_job(|done| OplogJob::AddPair { start, make_second, @@ -1784,7 +2055,10 @@ impl Oplog for PrimaryOplog { .await } - async fn commit(&self, level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + level: CommitLevel, + ) -> Result, OplogError> { self.run_job(|done| OplogJob::Commit { level, done }).await } @@ -1935,7 +2209,7 @@ impl Oplog for PrimaryOplog { &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result { + ) -> Result { // ORDERING (Start determinism): the job is enqueued synchronously here — there is no // `.await` between a subtask initiating its durable operation and this send — and the // actor assigns `Start` indices strictly in job order, so initiation order becomes @@ -1952,11 +2226,19 @@ impl Oplog for PrimaryOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { self.run_job(|done| OplogJob::AddIndexedStart { build_request, done, }) .await } + + fn shard_epoch(&self) -> Option { + self.shard_epoch + } + + fn fence(&self) -> Option { + self.fence.get().cloned() + } } diff --git a/golem-worker-executor/src/services/oplog/rate_limited.rs b/golem-worker-executor/src/services/oplog/rate_limited.rs index af0a76b9ce..ecf0e65d88 100644 --- a/golem-worker-executor/src/services/oplog/rate_limited.rs +++ b/golem-worker-executor/src/services/oplog/rate_limited.rs @@ -16,11 +16,12 @@ use crate::metrics::oplog::record_oplog_rate_limited; use crate::model::ExecutionStatus; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, Oplog, OplogAddReceipt, - OplogService, OrderedOplogStart, ReservedRawStartBuilder, + OplogError, OplogService, OrderedOplogStart, ReservedRawStartBuilder, }; use crate::services::resource_limits::{AtomicResourceEntry, ResourceLimits}; use arc_swap::ArcSwap; use async_trait::async_trait; +use golem_common::model::ShardEpoch; use golem_common::model::account::AccountId; use golem_common::model::agent::AgentMode; use golem_common::model::component::ComponentId; @@ -183,16 +184,16 @@ impl Oplog for RateLimitedOplog { let account_id = self.account_id; let environment_id = self.environment_id; Box::pin(async move { - let idx = pending.await; + let idx = pending.await?; Self::apply_rate_limit_for(&resource_entry, &state, &account_id, &environment_id).await; - idx + Ok(idx) }) } async fn add_durable_stream_batch( &self, make_batch: DurableStreamBatchBuilder, - ) -> Result, String> { + ) -> Result, OplogError> { let result = self.inner.add_durable_stream_batch(make_batch).await; self.apply_rate_limit().await; result @@ -202,7 +203,10 @@ impl Oplog for RateLimitedOplog { self.inner.drop_prefix(last_dropped_id).await } - async fn commit(&self, level: CommitLevel) -> BTreeMap { + async fn commit( + &self, + level: CommitLevel, + ) -> Result, OplogError> { self.inner.commit(level).await } @@ -267,18 +271,18 @@ impl Oplog for RateLimitedOplog { &self, start: OplogEntry, make_second: Box OplogEntry + Send>, - ) -> (OplogIndex, OplogIndex) { + ) -> Result<(OplogIndex, OplogIndex), OplogError> { // Assign the indices first, then throttle once for the pair: see `apply_rate_limit`. - let indices = self.inner.add_pair(start, make_second).await; + let indices = self.inner.add_pair(start, make_second).await?; self.apply_rate_limit().await; - indices + Ok(indices) } async fn add_start_with_reserved_raw_payload( &self, serialized_request: Vec, build_start: ReservedRawStartBuilder, - ) -> Result { + ) -> Result { // Order the `Start` first (the inner oplog assigns its index), then throttle. Applying // back-pressure before delegating would reorder concurrent calls' `Start` entries relative // to initiation order; see `apply_rate_limit`. @@ -293,7 +297,7 @@ impl Oplog for RateLimitedOplog { async fn add_start_with_indexed_reserved_raw_payload( &self, build_request: IndexedReservedStartBuilder, - ) -> Result { + ) -> Result { let ordered = self .inner .add_start_with_indexed_reserved_raw_payload(build_request) @@ -377,6 +381,7 @@ impl OplogService for RateLimitedOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { let account_id = initial_worker_metadata.created_by; let environment_id = owned_agent_id.environment_id; @@ -390,6 +395,7 @@ impl OplogService for RateLimitedOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ) .await; Arc::new(RateLimitedOplog::new( @@ -408,6 +414,7 @@ impl OplogService for RateLimitedOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { let account_id = initial_worker_metadata.created_by; let environment_id = owned_agent_id.environment_id; @@ -421,6 +428,7 @@ impl OplogService for RateLimitedOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ) .await; Arc::new(RateLimitedOplog::new( @@ -439,6 +447,7 @@ impl OplogService for RateLimitedOplogService { initial_worker_metadata: AgentMetadata, last_known_status: read_only_lock::arc_swap::ReadOnlyView, execution_status: read_only_lock::std::ReadOnlyLock, + shard_epoch: Option, ) -> Arc { let account_id = initial_worker_metadata.created_by; let environment_id = owned_agent_id.environment_id; @@ -452,6 +461,7 @@ impl OplogService for RateLimitedOplogService { initial_worker_metadata, last_known_status, execution_status, + shard_epoch, ) .await; Arc::new(RateLimitedOplog::new( @@ -672,6 +682,7 @@ mod tests { make_agent_metadata(agent_id, account_id, env_id), last_known_status, execution_status, + None, ) .await } @@ -696,7 +707,7 @@ mod tests { let start = Instant::now(); for _ in 0..15 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let elapsed = start.elapsed(); @@ -717,7 +728,7 @@ mod tests { let start = Instant::now(); for _ in 0..100 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let elapsed = start.elapsed(); @@ -735,7 +746,7 @@ mod tests { let start = Instant::now(); for _ in 0..100 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let elapsed = start.elapsed(); @@ -755,7 +766,7 @@ mod tests { // Unlimited — should be fast. let start = Instant::now(); for _ in 0..20 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let fast_elapsed = start.elapsed(); assert!( @@ -769,7 +780,7 @@ mod tests { // 15 writes at 5/sec (burst=5) must take >= 1.5 s. let start = Instant::now(); for _ in 0..15 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let slow_elapsed = start.elapsed(); assert!( @@ -788,7 +799,7 @@ mod tests { // 15 writes at 5/sec — must be slow. let start = Instant::now(); for _ in 0..15 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let slow_elapsed = start.elapsed(); assert!( @@ -802,7 +813,7 @@ mod tests { // 100 writes at unlimited — should be fast. let start = Instant::now(); for _ in 0..100 { - oplog.add(dummy_entry()).await; + oplog.add(dummy_entry()).await.unwrap(); } let fast_elapsed = start.elapsed(); assert!( @@ -868,7 +879,7 @@ mod tests { // An inline payload is already durable. small_pending.wait().await.unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); // Read back from storage (no in-memory cache) and confirm the large request is external and // its deferred blob upload became durable via the commit barrier. diff --git a/golem-worker-executor/src/services/oplog/tests.rs b/golem-worker-executor/src/services/oplog/tests.rs index 45595d32b9..2c30f444f0 100644 --- a/golem-worker-executor/src/services/oplog/tests.rs +++ b/golem-worker-executor/src/services/oplog/tests.rs @@ -28,6 +28,7 @@ use bytes::Bytes; use futures::FutureExt; use futures::stream::BoxStream; use golem_common::config::RedisConfig; +use golem_common::model::ShardEpoch; use golem_common::model::account::{AccountEmail, AccountId}; use golem_common::model::agent::{AgentMode, Principal}; use golem_common::model::card::{InvocationWalletPin, WalletVersionToken}; @@ -609,6 +610,35 @@ impl ReadCountingIndexedStorage { #[async_trait] impl IndexedStorage for ReadCountingIndexedStorage { + async fn upsert_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + shard_epoch: ShardEpoch, + ) -> Result<(), IndexedStorageError> { + self.inner + .upsert_oplog_metadata(svc_name, api_name, namespace, key, shard_epoch) + .await + } + + async fn delete_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + ) -> Result<(), IndexedStorageError> { + self.inner + .delete_oplog_metadata(svc_name, api_name, namespace, key) + .await + } + + fn supports_epoch_fencing(&self) -> bool { + self.inner.supports_epoch_fencing() + } + async fn number_of_replicas( &self, svc_name: &'static str, @@ -664,6 +694,7 @@ impl IndexedStorage for ReadCountingIndexedStorage { key: &str, id: u64, mut value: Vec, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.append_attempts.fetch_add(1, Ordering::Relaxed); if self.discard_compressed_appends @@ -687,7 +718,16 @@ impl IndexedStorage for ReadCountingIndexedStorage { value.push(0); } self.inner - .append(svc_name, api_name, entity_name, namespace, key, id, value) + .append( + svc_name, + api_name, + entity_name, + namespace, + key, + id, + value, + shard_epoch, + ) .await?; failure.after_write_result() } @@ -700,6 +740,7 @@ impl IndexedStorage for ReadCountingIndexedStorage { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.append_many_attempts.fetch_add(1, Ordering::Relaxed); if self.discard_compressed_appends @@ -745,7 +786,15 @@ impl IndexedStorage for ReadCountingIndexedStorage { pairs }; self.inner - .append_many(svc_name, api_name, entity_name, namespace, key, pairs) + .append_many( + svc_name, + api_name, + entity_name, + namespace, + key, + pairs, + shard_epoch, + ) .await?; failure.after_write_result() } @@ -1134,6 +1183,7 @@ async fn ephemeral_create_baseline_uses_lower_storage_and_checked_reads_find_it( metadata, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -1233,6 +1283,7 @@ async fn fresh_ephemeral_create_does_not_probe_lower_storage(_tracing: &Tracing) metadata, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -1326,6 +1377,7 @@ async fn fresh_ephemeral_create_with_compressed_layers_does_not_read_storage(_tr metadata, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -1392,6 +1444,7 @@ async fn primary_fresh_ephemeral_create_does_not_read_storage(_tracing: &Tracing metadata, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -1444,6 +1497,7 @@ async fn primary_uses_agent_mode_commit_threshold(_tracing: &Tracing) { metadata, default_last_known_status(), default_execution_status(agent_mode), + None, ) .await } @@ -1452,8 +1506,8 @@ async fn primary_uses_agent_mode_commit_threshold(_tracing: &Tracing) { let durable = open(AgentMode::Durable, "durable-threshold").await; let ephemeral = open(AgentMode::Ephemeral, "ephemeral-threshold").await; for oplog in [&durable, &ephemeral] { - oplog.add(OplogEntry::suspend().rounded()).await; - oplog.add(OplogEntry::exited().rounded()).await; + oplog.add(OplogEntry::suspend().rounded()).await.unwrap(); + oplog.add(OplogEntry::exited().rounded()).await.unwrap(); } assert_eq!(durable.length().await, 0); @@ -1519,6 +1573,7 @@ async fn fresh_ephemeral_create_with_blob_layers_does_not_read_storage(_tracing: metadata, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -1590,6 +1645,7 @@ async fn create_append_reconciliation_oplog( make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await } @@ -1638,6 +1694,7 @@ async fn explicit_commit_reports_threshold_commits_once_and_preserves_add_receip make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -1653,11 +1710,11 @@ async fn explicit_commit_reports_threshold_commits_once_and_preserves_add_receip .collect::>(); let mut expected = BTreeMap::new(); for (receipt, entry) in receipts.into_iter().zip(entries) { - expected.insert(receipt.await, entry); + expected.insert(receipt.await.unwrap(), entry); } - assert_eq!(oplog.commit(CommitLevel::Always).await, expected); - assert!(oplog.commit(CommitLevel::Always).await.is_empty()); + assert_eq!(oplog.commit(CommitLevel::Always).await.unwrap(), expected); + assert!(oplog.commit(CommitLevel::Always).await.unwrap().is_empty()); } #[test] @@ -1703,6 +1760,7 @@ async fn archiving_auto_committed_entries_does_not_consume_explicit_commit_repor make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -1713,21 +1771,21 @@ async fn archiving_auto_committed_entries_does_not_consume_explicit_commit_repor ]; let mut expected = BTreeMap::new(); for entry in entries { - let index = oplog.add(entry.clone()).await; + let index = oplog.add(entry.clone()).await.unwrap(); expected.insert(index, entry); } MultiLayerOplog::try_archive_blocking(&oplog).await; - assert_eq!(oplog.commit(CommitLevel::Always).await, expected); - assert!(oplog.commit(CommitLevel::Always).await.is_empty()); + assert_eq!(oplog.commit(CommitLevel::Always).await.unwrap(), expected); + assert!(oplog.commit(CommitLevel::Always).await.unwrap().is_empty()); let mut before_commit = BTreeMap::new(); for entry in [ OplogEntry::interrupted().rounded(), OplogEntry::resumed().rounded(), ] { - before_commit.insert(oplog.add(entry.clone()).await, entry); + before_commit.insert(oplog.add(entry.clone()).await.unwrap(), entry); } let mut commit = std::pin::pin!(oplog.commit(CommitLevel::Always)); assert!(futures::poll!(commit.as_mut()).is_pending()); @@ -1739,12 +1797,15 @@ async fn archiving_auto_committed_entries_does_not_consume_explicit_commit_repor OplogEntry::suspend().rounded(), OplogEntry::restart().rounded(), ] { - after_commit.insert(oplog.add(entry.clone()).await, entry); + after_commit.insert(oplog.add(entry.clone()).await.unwrap(), entry); } - assert_eq!(commit.await, before_commit); + assert_eq!(commit.await.unwrap(), before_commit); MultiLayerOplog::try_archive_blocking(&oplog).await; - assert_eq!(oplog.commit(CommitLevel::Always).await, after_commit); - assert!(oplog.commit(CommitLevel::Always).await.is_empty()); + assert_eq!( + oplog.commit(CommitLevel::Always).await.unwrap(), + after_commit + ); + assert!(oplog.commit(CommitLevel::Always).await.unwrap().is_empty()); } #[test] @@ -1767,14 +1828,14 @@ async fn wait_for_replicas_does_not_consume_explicit_commit_report(_tracing: &Tr ]; let mut expected = BTreeMap::new(); for entry in entries { - let index = oplog.add(entry.clone()).await; + let index = oplog.add(entry.clone()).await.unwrap(); expected.insert(index, entry); } assert!(oplog.wait_for_replicas(1, Duration::from_secs(1)).await); - assert_eq!(oplog.commit(CommitLevel::Always).await, expected); - assert!(oplog.commit(CommitLevel::Always).await.is_empty()); + assert_eq!(oplog.commit(CommitLevel::Always).await.unwrap(), expected); + assert!(oplog.commit(CommitLevel::Always).await.unwrap().is_empty()); } #[test] @@ -1799,9 +1860,12 @@ async fn retried_append_many_accepts_only_the_same_serialized_batch(_tracing: &T indexed_storage.reset_append_observations(); indexed_storage.inject_append_many_failure(InjectedAppendFailure::CommitThenIndeterminate); - oplog.add(OplogEntry::suspend()).await; - oplog.add(OplogEntry::exited()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + oplog.add(OplogEntry::exited()).await.expect("oplog write"); + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!(oplog.current_oplog_index().await, OplogIndex::from_u64(3)); assert_eq!(indexed_storage.append_many_attempts(), 1); @@ -1819,8 +1883,11 @@ async fn indeterminate_append_before_write_retries_after_empty_read_back(_tracin indexed_storage.inject_append_many_failure(InjectedAppendFailure::IndeterminateBeforeWrite); let entry = OplogEntry::suspend().rounded(); - oplog.add(entry.clone()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(entry.clone()).await.expect("oplog write"); + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!(indexed_storage.append_many_attempts(), 2); assert_eq!(indexed_storage.reads(), 1); @@ -1842,8 +1909,11 @@ async fn exhausted_retries_after_committed_indeterminate_append_reconcile(_traci InjectedAppendFailure::TransientBeforeWrite, ]); - oplog.add(OplogEntry::suspend()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!(indexed_storage.append_many_attempts(), 3); assert_eq!(indexed_storage.reads(), 3); @@ -1864,8 +1934,11 @@ async fn permanent_retry_failure_after_committed_indeterminate_append_reconciles InjectedAppendFailure::PermanentBeforeWrite, ]); - oplog.add(OplogEntry::suspend()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!(indexed_storage.append_many_attempts(), 2); assert_eq!(indexed_storage.reads(), 2); @@ -1888,8 +1961,11 @@ async fn reconciliation_retries_read_failures_without_resubmitting_append(_traci "connection lost during read-back".to_string(), )); - oplog.add(OplogEntry::suspend()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!(indexed_storage.append_many_attempts(), 1); assert_eq!(indexed_storage.reads(), 2); @@ -1905,8 +1981,11 @@ async fn conflict_after_initially_empty_reconciliation_accepts_exact_batch(_trac indexed_storage.inject_append_many_failure(InjectedAppendFailure::CommitThenIndeterminate); indexed_storage.hidden_reads.store(1, Ordering::Relaxed); - oplog.add(OplogEntry::suspend()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!(indexed_storage.append_many_attempts(), 2); assert_eq!(indexed_storage.reads(), 2); @@ -1939,6 +2018,7 @@ async fn direct_identical_append_conflict_from_second_writer_remains_fatal(_trac make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let second_oplog = second_service @@ -1949,12 +2029,16 @@ async fn direct_identical_append_conflict_from_second_writer_remains_fatal(_trac make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let entry = OplogEntry::suspend(); - first_oplog.add(entry.clone()).await; - second_oplog.add(entry).await; - first_oplog.commit(CommitLevel::Always).await; + first_oplog.add(entry.clone()).await.expect("oplog write"); + second_oplog.add(entry).await.expect("oplog write"); + first_oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); indexed_storage.reset(); indexed_storage.reset_append_observations(); @@ -1974,8 +2058,8 @@ async fn incomplete_read_back_after_indeterminate_append_remains_fatal(_tracing: indexed_storage .inject_append_many_failure(InjectedAppendFailure::CommitPrefixThenIndeterminate); - oplog.add(OplogEntry::suspend()).await; - oplog.add(OplogEntry::exited()).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + oplog.add(OplogEntry::exited()).await.expect("oplog write"); assert_panics(oplog.commit(CommitLevel::Always)).await; assert_eq!(indexed_storage.append_many_attempts(), 1); @@ -1992,7 +2076,7 @@ async fn differing_read_back_after_indeterminate_append_remains_fatal(_tracing: indexed_storage .inject_append_many_failure(InjectedAppendFailure::CommitDifferentThenIndeterminate); - oplog.add(OplogEntry::suspend()).await; + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); assert_panics(oplog.commit(CommitLevel::Always)).await; assert_eq!(indexed_storage.append_many_attempts(), 1); @@ -2027,6 +2111,7 @@ async fn open_add_and_read_back(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -2042,10 +2127,10 @@ async fn open_add_and_read_back(_tracing: &Tracing) { let entry3 = OplogEntry::exited().rounded(); let last_oplog_idx = oplog.current_oplog_index().await; - oplog.add(entry1.clone()).await; - oplog.add(entry2.clone()).await; - oplog.add(entry3.clone()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(entry1.clone()).await.unwrap(); + oplog.add(entry2.clone()).await.unwrap(); + oplog.add(entry3.clone()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); let r1 = oplog.read(last_oplog_idx.next()).await; let r2 = oplog.read(last_oplog_idx.next().next()).await; @@ -2101,6 +2186,7 @@ async fn primary_read_range_overflow_panics_without_storage_io(_tracing: &Tracin make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; assert_panics(oplog.read_exact(start, 2)).await; @@ -2145,6 +2231,7 @@ async fn primary_storage_read_failures_panic_from_all_read_paths(_tracing: &Trac make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; assert_panics(oplog.read_exact(OplogIndex::INITIAL, 1)).await; @@ -2158,6 +2245,7 @@ async fn primary_storage_read_failures_panic_from_all_read_paths(_tracing: &Trac make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; assert_panics(oplog.read(OplogIndex::INITIAL)).await; @@ -2239,6 +2327,7 @@ async fn durable_stream_batch_externalizes_every_record_family(_tracing: &Tracin make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let stream_id = StreamId(Uuid::new_v4()); @@ -2322,7 +2411,7 @@ async fn durable_stream_batch_externalizes_every_record_family(_tracing: &Tracin })) .await .unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); assert_eq!(added.len(), 4); for (_, entry) in added { @@ -2422,6 +2511,7 @@ async fn durable_stream_producer_recovers_from_sqlite_storage_restart(_tracing: make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let producer = DurableStreamProducer::load( @@ -2468,6 +2558,7 @@ async fn durable_stream_producer_recovers_from_sqlite_storage_restart(_tracing: make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let restarted = DurableStreamProducer::load( @@ -2543,6 +2634,7 @@ async fn open_add_and_read_back_many(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -2559,12 +2651,12 @@ async fn open_add_and_read_back_many(_tracing: &Tracing) { let entry4 = OplogEntry::interrupted().rounded(); let entry5 = OplogEntry::no_op(None).rounded(); - oplog.add(entry1.clone()).await; - oplog.add(entry2.clone()).await; - oplog.add(entry3.clone()).await; - oplog.commit(CommitLevel::Always).await; - oplog.add(entry4.clone()).await; - oplog.add(entry5.clone()).await; // uncommitted entries + oplog.add(entry1.clone()).await.unwrap(); + oplog.add(entry2.clone()).await.unwrap(); + oplog.add(entry3.clone()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); + oplog.add(entry4.clone()).await.unwrap(); + oplog.add(entry5.clone()).await.unwrap(); // uncommitted entries let read_count = indexed_storage.read_count(); let buffered_entries = oplog @@ -2653,6 +2745,7 @@ async fn open_add_and_read_back_ephemeral(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -2668,10 +2761,10 @@ async fn open_add_and_read_back_ephemeral(_tracing: &Tracing) { let entry3 = OplogEntry::exited().rounded(); let last_oplog_idx = oplog.current_oplog_index().await; - oplog.add(entry1.clone()).await; - oplog.add(entry2.clone()).await; - oplog.add(entry3.clone()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(entry1.clone()).await.unwrap(); + oplog.add(entry2.clone()).await.unwrap(); + oplog.add(entry3.clone()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); let r1 = oplog.read(last_oplog_idx.next()).await; let r2 = oplog.read(last_oplog_idx.next().next()).await; @@ -2744,6 +2837,7 @@ async fn open_add_and_read_back_many_ephemeral(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -2759,11 +2853,11 @@ async fn open_add_and_read_back_many_ephemeral(_tracing: &Tracing) { let entry3 = OplogEntry::exited().rounded(); let entry4 = OplogEntry::interrupted().rounded(); - oplog.add(entry1.clone()).await; - oplog.add(entry2.clone()).await; - oplog.add(entry3.clone()).await; - oplog.commit(CommitLevel::Always).await; - oplog.add(entry4.clone()).await; // uncommitted + oplog.add(entry1.clone()).await.unwrap(); + oplog.add(entry2.clone()).await.unwrap(); + oplog.add(entry3.clone()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); + oplog.add(entry4.clone()).await.unwrap(); // uncommitted let entries = oplog .read_exact(OplogIndex::INITIAL, 4) @@ -2816,6 +2910,7 @@ async fn ephemeral_read_exact_committed_only(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -2823,10 +2918,10 @@ async fn ephemeral_read_exact_committed_only(_tracing: &Tracing) { let entry2 = OplogEntry::exited().rounded(); let entry3 = OplogEntry::interrupted().rounded(); - oplog.add(entry1.clone()).await; - oplog.add(entry2.clone()).await; - oplog.add(entry3.clone()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(entry1.clone()).await.unwrap(); + oplog.add(entry2.clone()).await.unwrap(); + oplog.add(entry3.clone()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); // All committed, no buffer entries let entries = oplog @@ -2880,14 +2975,15 @@ async fn ephemeral_read_exact_uncommitted_only(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; let entry1 = OplogEntry::suspend().rounded(); let entry2 = OplogEntry::exited().rounded(); - oplog.add(entry1.clone()).await; - oplog.add(entry2.clone()).await; + oplog.add(entry1.clone()).await.unwrap(); + oplog.add(entry2.clone()).await.unwrap(); // No commit — entries only in the buffer let entries = oplog @@ -2941,6 +3037,7 @@ async fn ephemeral_read_exact_partial_range(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -2956,16 +3053,16 @@ async fn ephemeral_read_exact_partial_range(_tracing: &Tracing) { retry_policy_state: None, } .rounded(); - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); entries.push(entry); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); // Add 2 more uncommitted let uncommitted1 = OplogEntry::interrupted().rounded(); let uncommitted2 = OplogEntry::suspend().rounded(); - oplog.add(uncommitted1.clone()).await; - oplog.add(uncommitted2.clone()).await; + oplog.add(uncommitted1.clone()).await.unwrap(); + oplog.add(uncommitted2.clone()).await.unwrap(); entries.push(uncommitted1); entries.push(uncommitted2); @@ -3044,6 +3141,7 @@ async fn ephemeral_read_exact_across_archive_layers(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -3065,15 +3163,15 @@ async fn ephemeral_read_exact_across_archive_layers(_tracing: &Tracing) { let initial_oplog_idx = oplog.current_oplog_index().await; for entry in &entries { - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); // Add 2 uncommitted entries let uncommitted1 = OplogEntry::interrupted().rounded(); let uncommitted2 = OplogEntry::suspend().rounded(); - oplog.add(uncommitted1.clone()).await; - oplog.add(uncommitted2.clone()).await; + oplog.add(uncommitted1.clone()).await.unwrap(); + oplog.add(uncommitted2.clone()).await.unwrap(); entries.push(uncommitted1); entries.push(uncommitted2); @@ -3168,10 +3266,11 @@ async fn ephemeral_read_exact_zero_returns_empty(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; - oplog.add(OplogEntry::suspend().rounded()).await; + oplog.add(OplogEntry::suspend().rounded()).await.unwrap(); let entries = oplog.read_exact(OplogIndex::INITIAL, 0).await; assert!(entries.is_empty()); @@ -3206,6 +3305,7 @@ async fn entries_with_small_payload(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -3267,9 +3367,9 @@ async fn entries_with_small_payload(_tracing: &Tracing) { description: desc.clone(), } .rounded(); - oplog.add(entry4.clone()).await; + oplog.add(entry4.clone()).await.unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let r_start = oplog.read(last_oplog_idx.next()).await.rounded(); let r_end = oplog.read(last_oplog_idx.next().next()).await.rounded(); @@ -3409,6 +3509,7 @@ async fn completed_host_call_response_upload_failure_writes_no_start(_tracing: & make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let before = oplog.current_oplog_index().await; @@ -3488,6 +3589,7 @@ async fn owned_invocation_payload_upload_failure_writes_no_entry(_tracing: &Trac make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let before = oplog.current_oplog_index().await; @@ -3541,6 +3643,7 @@ async fn entries_with_large_payload(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -3613,9 +3716,9 @@ async fn entries_with_large_payload(_tracing: &Tracing) { description: desc.clone(), } .rounded(); - oplog.add(entry4.clone()).await; + oplog.add(entry4.clone()).await.unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let r_start = oplog.read(last_oplog_idx.next()).await.rounded(); let r_end = oplog.read(last_oplog_idx.next().next()).await.rounded(); @@ -3827,6 +3930,7 @@ async fn multilayer_transfers_entries_after_limit_reached( make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let mut entries = Vec::new(); @@ -3848,8 +3952,8 @@ async fn multilayer_transfers_entries_after_limit_reached( durable_function_type: DurableFunctionType::ReadLocal, } .rounded(); - oplog.add(entry.clone()).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(entry.clone()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); entries.push(entry); } @@ -3863,6 +3967,7 @@ async fn multilayer_transfers_entries_after_limit_reached( make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -3894,6 +3999,7 @@ async fn multilayer_transfers_entries_after_limit_reached( make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -3988,6 +4094,7 @@ async fn read_from_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -4009,13 +4116,13 @@ async fn read_from_archive_impl(use_blob: bool) { let initial_oplog_idx = oplog.current_oplog_index().await; for entry in &entries { - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let uncommitted1 = OplogEntry::interrupted().rounded(); let uncommitted2 = OplogEntry::suspend().rounded(); - oplog.add(uncommitted1.clone()).await; - oplog.add(uncommitted2.clone()).await; + oplog.add(uncommitted1.clone()).await.unwrap(); + oplog.add(uncommitted2.clone()).await.unwrap(); entries.push(uncommitted1); entries.push(uncommitted2); @@ -4030,6 +4137,7 @@ async fn read_from_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -4177,6 +4285,7 @@ async fn read_initial_from_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -4311,9 +4420,10 @@ async fn ephemeral_read_initial_from_archive_impl(use_blob: bool) { }, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let read_before_archive = oplog_service .read_exact( @@ -4613,6 +4723,7 @@ async fn open_multilayer_oplog_retains_stale_index_after_service_deletion(_traci make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let current = oplog.current_oplog_index().await; @@ -4708,11 +4819,12 @@ async fn deleting_worker_fences_in_flight_archive_transfers_impl(agent_mode: Age }, default_last_known_status(), default_execution_status(agent_mode), + None, ) .await; - oplog.add(OplogEntry::no_op(None)).await; - oplog.commit(CommitLevel::Always).await; + oplog.add(OplogEntry::no_op(None)).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); if agent_mode == AgentMode::Ephemeral { EphemeralOplog::try_archive(&oplog) .await @@ -4802,6 +4914,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; info!("FIRST OPEN DONE"); @@ -4824,9 +4937,9 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { let initial_oplog_idx = oplog.current_oplog_index().await; for entry in &entries { - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); tokio::time::sleep(Duration::from_secs(2)).await; let primary_length = primary_oplog_service @@ -4837,6 +4950,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -4867,6 +4981,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await } else if reopen == Reopen::Full { @@ -4896,6 +5011,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await } else { @@ -4917,12 +5033,12 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { .collect(); for (n, entry) in entries.iter().enumerate() { - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); if n % 100 == 0 { - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); } } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); tokio::time::sleep(Duration::from_secs(2)).await; let primary_length = primary_oplog_service @@ -4933,6 +5049,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -4963,6 +5080,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await } else if reopen == Reopen::Full { @@ -4992,6 +5110,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await } else { @@ -5010,8 +5129,9 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { } .rounded(), ) - .await; - oplog.commit(CommitLevel::Always).await; + .await + .unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); drop(oplog); let entry1 = oplog_service @@ -5161,6 +5281,7 @@ async fn empty_layer_gets_deleted_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -5185,9 +5306,9 @@ async fn empty_layer_gets_deleted_impl(use_blob: bool) { .collect(); for entry in &entries { - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); tokio::time::sleep(Duration::from_millis(100)).await; } @@ -5211,6 +5332,7 @@ async fn empty_layer_gets_deleted_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -5320,12 +5442,13 @@ async fn scheduled_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; for entry in &entries { - oplog.add(entry.clone()).await; + oplog.add(entry.clone()).await.unwrap(); } - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let result = MultiLayerOplog::try_archive(&oplog).await; drop(oplog); @@ -5346,6 +5469,7 @@ async fn scheduled_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -5386,6 +5510,7 @@ async fn scheduled_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let result = MultiLayerOplog::try_archive(&oplog).await; @@ -5403,6 +5528,7 @@ async fn scheduled_archive_impl(use_blob: bool) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await .length() @@ -5499,6 +5625,7 @@ async fn multilayer_scan_for_component(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -5519,7 +5646,8 @@ async fn multilayer_scan_for_component(_tracing: &Tracing) { "test".to_string(), "test".to_string(), )) - .await; + .await + .unwrap(); } } 2 => { @@ -5538,7 +5666,8 @@ async fn multilayer_scan_for_component(_tracing: &Tracing) { "test".to_string(), "test".to_string(), )) - .await; + .await + .unwrap(); } debug!("[{r:?}] => archiving {agent_id} to tertiary layer"); @@ -5553,7 +5682,8 @@ async fn multilayer_scan_for_component(_tracing: &Tracing) { "test".to_string(), "test".to_string(), )) - .await; + .await + .unwrap(); } } _ => unreachable!(), @@ -5657,9 +5787,10 @@ async fn multilayer_scan_for_component_ephemeral(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(mode), + None, ) .await; - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); owned_agent_id }; @@ -5767,9 +5898,10 @@ async fn concurrent_get_or_open_does_not_cause_unique_key_violation(_tracing: &T make_agent_metadata(worker_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; - initial_oplog.commit(CommitLevel::Always).await; + initial_oplog.commit(CommitLevel::Always).await.unwrap(); drop(initial_oplog); // Wait for the weak reference to become invalid so the cache entry is evicted @@ -5806,6 +5938,7 @@ async fn concurrent_get_or_open_does_not_cause_unique_key_violation(_tracing: &T make_agent_metadata(worker_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -5813,10 +5946,10 @@ async fn concurrent_get_or_open_does_not_cause_unique_key_violation(_tracing: &T // different oplog instances (due to the get_or_open race), they'll // have independent last_committed_idx and produce duplicate ids, // causing a unique key violation on commit. - oplog.add(OplogEntry::suspend()).await; - // Use fallible_add pattern: commit can panic on unique key violation; + oplog.add(OplogEntry::suspend()).await.unwrap(); + // `add` is fallible now: commit can panic on unique key violation; // we use the Oplog trait method directly and let it propagate. - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); tokio::task::yield_now().await; } @@ -6021,6 +6154,7 @@ async fn durable_and_ephemeral_oplogs_are_isolated_for_same_agent_id(_tracing: & make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; let ephemeral_oplog = oplog_service @@ -6031,10 +6165,11 @@ async fn durable_and_ephemeral_oplogs_are_isolated_for_same_agent_id(_tracing: & make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; - durable_oplog.commit(CommitLevel::Always).await; - ephemeral_oplog.commit(CommitLevel::Always).await; + durable_oplog.commit(CommitLevel::Always).await.unwrap(); + ephemeral_oplog.commit(CommitLevel::Always).await.unwrap(); // Both namespaces report the oplog exists, independently. assert!( @@ -6129,9 +6264,10 @@ async fn make_workers( make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(mode), + None, ) .await; - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); out.push(owned_agent_id); } out @@ -6386,6 +6522,7 @@ async fn owned_payload_upload_preserves_allocation_at_inline_threshold_and_round make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -6429,6 +6566,7 @@ async fn owned_payload_upload_preserves_allocation_at_inline_threshold_and_round make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; assert_eq!( @@ -6482,6 +6620,7 @@ async fn owned_snapshot_payloads_persist_and_replay_across_inline_threshold(_tra make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -6531,14 +6670,16 @@ async fn owned_snapshot_payloads_persist_and_replay_across_inline_threshold(_tra timestamp: Timestamp::now_utc(), description: inline_description, }) - .await; + .await + .unwrap(); let external_index = oplog .add(OplogEntry::PendingUpdate { timestamp: Timestamp::now_utc(), description: external_description, }) - .await; - oplog.commit(CommitLevel::Always).await; + .await + .unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); let persisted = oplog_service .read_exact(&owned_agent_id, AgentMode::Durable, inline_index, 2) @@ -6614,6 +6755,7 @@ async fn reserved_large_request_is_durable_via_commit_barrier(_tracing: &Tracing make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -6638,7 +6780,7 @@ async fn reserved_large_request_is_durable_via_commit_barrier(_tracing: &Tracing .unwrap(); assert_eq!(start_idx, last_oplog_idx.next()); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); // Read back from the service (storage), so the payload reference carries no in-memory cache and // the download must hit blob storage. @@ -6702,6 +6844,7 @@ async fn reserved_small_request_stays_inline(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -6724,7 +6867,7 @@ async fn reserved_small_request_stays_inline(_tracing: &Tracing) { // Inline payloads are already durable: waiting is a no-op. pending.wait().await.unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let entries = oplog_service .read_exact(&owned_agent_id, AgentMode::Durable, start_idx, 1) @@ -6826,6 +6969,7 @@ async fn multilayer_reserved_start_delegates_to_primary_and_tracks_last_index(_t make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -6854,7 +6998,7 @@ async fn multilayer_reserved_start_delegates_to_primary_and_tracks_last_index(_t first_pending.wait().await.unwrap(); second_pending.wait().await.unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); let entries = oplog_service .read_exact(&owned_agent_id, AgentMode::Durable, first_idx, 2) @@ -6930,6 +7074,7 @@ async fn ephemeral_reserved_start_uploads_payload_eagerly(_tracing: &Tracing) { metadata, default_last_known_status(), default_execution_status(AgentMode::Ephemeral), + None, ) .await; @@ -7147,6 +7292,7 @@ async fn reserved_start_through_production_stack_smoke(_tracing: &Tracing) { make_agent_metadata(agent_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), + None, ) .await; @@ -7177,7 +7323,7 @@ async fn reserved_start_through_production_stack_smoke(_tracing: &Tracing) { assert_eq!(small_idx, large_idx.next()); small_pending.wait().await.unwrap(); - oplog.commit(CommitLevel::Always).await; + oplog.commit(CommitLevel::Always).await.unwrap(); // Read back through the service stack (no in-memory cache). let entries = oplog_service @@ -7239,3 +7385,434 @@ async fn reserved_start_through_production_stack_smoke(_tracing: &Tracing) { other => panic!("unexpected request: {other:?}"), } } + +/// The fence, end to end through the real oplog service and a backend that enforces it. +/// +/// SQLite rather than the in-memory backend on purpose: in-memory does not fence, so it would +/// pass these no matter what the service does. +async fn fencing_oplog_service(tempdir: &tempfile::TempDir, name: &str) -> PrimaryOplogService { + let config = golem_common::config::DbSqliteConfig { + database: tempdir + .path() + .join(format!("{name}.db")) + .to_string_lossy() + .into_owned(), + max_connections: 4, + foreign_keys: false, + }; + let indexed_storage: Arc = + Arc::new(SqliteIndexedStorage::configured(&config).await.unwrap()); + assert!( + indexed_storage.supports_epoch_fencing(), + "this test is meaningless on a backend that cannot fence" + ); + PrimaryOplogService::new( + indexed_storage, + Arc::new(InMemoryBlobStorage::new()), + 100, + 1, + 128, + RetryConfig::default(), + ) + .await +} + +#[test] +async fn an_oplog_opened_at_the_owning_epoch_can_be_written(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let service = fencing_oplog_service(&tempdir, "owning").await; + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "owned".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + + let oplog = service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(7)), + ) + .await; + + // Opening records the epoch, so the writes that follow are accepted. + oplog.add(OplogEntry::suspend().rounded()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); + assert_eq!(oplog.length().await, 1); +} + +#[test] +async fn an_oplog_opened_at_a_stale_epoch_refuses_every_write(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "moved".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + + // Two services over one database, because that is what two executors are. A single service + // would not do: `OpenOplogs` caches by agent id, so a second `open` on it hands back the + // first oplog - epoch and all - instead of constructing a new one. + let owning_executor = fencing_oplog_service(&tempdir, "shared").await; + let losing_executor = fencing_oplog_service(&tempdir, "shared").await; + + // The shard's new owner takes it over at a higher epoch and writes. + let owner = owning_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(9)), + ) + .await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + + // The executor that lost the shard still believes it holds epoch 8. It is refused at its + // very first write, and told which epoch owns the oplog now. + let loser = losing_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(8)), + ) + .await; + // `add` only buffers; the storage write happens at the commit, so the refusal is asserted + // over the pair rather than over `add` alone. + let write = async { + loser.add(OplogEntry::exited().rounded()).await?; + loser.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + match write { + Err(OplogError::Fenced(fence)) => { + assert_eq!(fence.agent_id, agent_id); + assert_eq!(fence.expected_epoch, golem_common::model::ShardEpoch(8)); + assert_eq!( + fence.actual_epoch, + Some(golem_common::model::ShardEpoch(9)), + "the fence must name the epoch that owns the oplog now" + ); + } + other => panic!("expected the write to be fenced, got {other:?}"), + } + + // ... and stays refused: the oplog is poisoned, so it does not even ask the storage again. + assert!(matches!( + loser.commit(CommitLevel::Always).await, + Err(OplogError::Fenced(_)) + )); + assert_eq!( + owner.length().await, + 1, + "the losing executor must not have appended to the owner's oplog" + ); +} + +#[test] +async fn an_oplog_opened_without_an_epoch_asserts_nothing(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let service = fencing_oplog_service(&tempdir, "unfenced").await; + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "unfenced".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + + // `None` is what the debugging service and a fork of a remote target pass: no ownership + // claim, so the record is neither written nor checked. + let oplog = service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + None, + ) + .await; + oplog.add(OplogEntry::suspend().rounded()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); + assert_eq!(oplog.length().await, 1); +} + +#[test] +async fn deleting_an_oplog_fences_a_writer_that_still_holds_it(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let service = fencing_oplog_service(&tempdir, "deleted").await; + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "deleted".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + + let oplog = service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(7)), + ) + .await; + oplog.add(OplogEntry::suspend().rounded()).await.unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); + + service.delete(&owned_agent_id, AgentMode::Durable).await; + + // The handle outlives the delete, as a zombie executor's would. Its epoch is still the one + // the record held, so only the record's absence can refuse it - an entry landing here would + // bring back an oplog that was deleted. + let write = async { + oplog.add(OplogEntry::exited().rounded()).await?; + oplog.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + match write { + Err(OplogError::Fenced(fence)) => { + assert_eq!(fence.agent_id, agent_id); + assert_eq!(fence.expected_epoch, golem_common::model::ShardEpoch(7)); + assert_eq!( + fence.actual_epoch, None, + "the record must be gone, not moved to another epoch" + ); + } + other => panic!("expected the write to be fenced, got {other:?}"), + } + assert!( + !service.exists(&owned_agent_id, AgentMode::Durable).await, + "the refused write must not have brought the deleted oplog back" + ); +} + +#[test] +async fn a_fenced_oplog_is_not_handed_out_again_while_it_is_still_held(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "regained".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let executor = fencing_oplog_service(&tempdir, "shared").await; + let other_executor = fencing_oplog_service(&tempdir, "shared").await; + let open = |service: &PrimaryOplogService, epoch: u64| { + let service = service.clone(); + let agent_id = agent_id.clone(); + let owned_agent_id = owned_agent_id.clone(); + async move { + service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(epoch)), + ) + .await + } + }; + + // This executor holds the shard at epoch 8, loses it to the other executor at 9, and is + // refused - the handle is fenced and stays open, as a worker still stopping keeps it. + let fenced = open(&executor, 8).await; + let owner = open(&other_executor, 9).await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + fenced.add(OplogEntry::exited().rounded()).await.unwrap(); + assert!(matches!( + fenced.commit(CommitLevel::Always).await, + Err(OplogError::Fenced(_)) + )); + drop(owner); + + // Re-granted the shard at 10 while the fenced handle is still alive, the executor opens the + // agent again - recovering it - and must not be handed the finished handle: that one refuses + // every write, and its view of the oplog stops where its refused entries began. + let regained = open(&executor, 10).await; + assert!( + !Arc::ptr_eq(®ained, &fenced), + "the fenced handle was handed out again" + ); + regained.add(OplogEntry::exited().rounded()).await.unwrap(); + regained.commit(CommitLevel::Always).await.unwrap(); + assert_eq!(regained.length().await, 2); + + // Dropping the fenced handle runs its remover; it must not evict the fresh handle, or the + // next open would construct a third, and two live writers would share one oplog. + drop(fenced); + let again = open(&executor, 10).await; + assert!( + Arc::ptr_eq(&again, ®ained), + "the fresh handle was evicted by the fenced handle's removal" + ); +} + +#[test] +async fn an_invocation_started_records_the_epoch_its_oplog_asserts(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let primary = Arc::new(fencing_oplog_service(&tempdir, "recorded").await); + let archive: Arc = Arc::new(CompressedOplogArchiveService::new( + Arc::new(InMemoryIndexedStorage::new()), + 1, + RetryConfig::default(), + )); + // Through the layered service rather than the primary alone: the entry is written from the + // top of a wrapper chain, as in production, so the epoch has to be found through it. + let service = MultiLayerOplogService::new(primary, nev![archive], 10, 10); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + + // `None` must stay `None` rather than become epoch 0, which is a real epoch. + for (name, shard_epoch) in [ + ("fenced", Some(golem_common::model::ShardEpoch(9))), + ("unfenced", None), + ] { + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: name.into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let oplog = service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + shard_epoch, + ) + .await; + + let index = oplog + .add_agent_invocation_started_with_index( + AgentInvocation::AgentMethod { + idempotency_key: IdempotencyKey::fresh(), + method_name: "f".to_string(), + input: SchemaValue::Record { fields: vec![] }, + invocation_context: InvocationContextStack::fresh_rounded(), + principal: Principal::anonymous(), + scope_card: None, + }, + invocation_wallet_pin(), + ) + .await + .unwrap(); + oplog.commit(CommitLevel::Always).await.unwrap(); + + // Read back from the storage, so the field is shown to survive being persisted as well. + let stored = service + .read_exact(&owned_agent_id, AgentMode::Durable, index, 1) + .await + .remove(&index) + .expect("the committed entry reads back"); + match stored { + OplogEntry::AgentInvocationStarted { + shard_epoch: recorded, + .. + } => assert_eq!( + recorded, + shard_epoch.map(|epoch| epoch.0), + "{name}: the entry must record the epoch its oplog asserts" + ), + other => panic!("{name}: expected AgentInvocationStarted, got {other:?}"), + } + } +} + +#[test] +async fn an_executor_that_loses_the_shard_mid_flight_is_refused_at_its_next_write( + _tracing: &Tracing, +) { + // The realistic sequence, and the one only the per-write assertion can catch: this executor + // opened the oplog while it still owned the shard, so its epoch record went in cleanly and + // nothing was poisoned at open. The shard moves underneath it afterwards. + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "mid-flight".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + + let losing_executor = fencing_oplog_service(&tempdir, "mid-flight").await; + let gaining_executor = fencing_oplog_service(&tempdir, "mid-flight").await; + + let loser = losing_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(4)), + ) + .await; + loser.add(OplogEntry::suspend().rounded()).await.unwrap(); + loser.commit(CommitLevel::Always).await.unwrap(); + assert_eq!(loser.length().await, 1, "it owned the shard at this point"); + + // The shard is re-granted to another executor, which opens the oplog at the new epoch. + let _gainer = gaining_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(5)), + ) + .await; + + // The loser's already-open oplog is not poisoned - it had no reason to be - so this is the + // per-write epoch assertion doing the work, and nothing of its is written. + let write = async { + loser.add(OplogEntry::exited().rounded()).await?; + loser.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + match write { + Err(OplogError::Fenced(fence)) => { + assert_eq!(fence.expected_epoch, golem_common::model::ShardEpoch(4)); + assert_eq!(fence.actual_epoch, Some(golem_common::model::ShardEpoch(5))); + } + other => panic!("expected the in-flight write to be fenced, got {other:?}"), + } + assert_eq!( + loser.length().await, + 1, + "the refused entry must not be there" + ); +} diff --git a/golem-worker-executor/src/services/rpc.rs b/golem-worker-executor/src/services/rpc.rs index 46941f36c8..b3bea1a721 100644 --- a/golem-worker-executor/src/services/rpc.rs +++ b/golem-worker-executor/src/services/rpc.rs @@ -762,6 +762,8 @@ fn rpc_error_from_rejection(rejected: InvocationRejected) -> RpcError { InvocationRejectionReason::Internal => RpcError::RemoteInternalError { details: rejected.error, }, + // The routing miss an executor reports as a typed failure once it has accepted. + InvocationRejectionReason::ShardingNotReady => WorkerExecutorError::ShardingNotReady.into(), _ => RpcError::ProtocolError { details: rejected.error, }, diff --git a/golem-worker-executor/src/services/shard_manager.rs b/golem-worker-executor/src/services/shard_manager.rs index d36f4c1c37..67d7b50d0e 100644 --- a/golem-worker-executor/src/services/shard_manager.rs +++ b/golem-worker-executor/src/services/shard_manager.rs @@ -105,6 +105,13 @@ pub trait ShardManagerService: Send + Sync { /// recovery only if nothing newer was registered since: a newer registration means a delivery /// this attempt did not see, whose agents it therefore did not start. No-op by default. fn recovery_succeeded(&self, _ticket: RecoveryTicket) {} + + /// Whether shards can move between executors under this implementation, and therefore + /// whether the oplog fence has to exist for the executor to be safe. + /// + /// Deliberately without a default: the answer decides whether the process is allowed to + /// start at all, so a new implementation must state it rather than inherit a permissive one. + fn requires_oplog_fencing(&self) -> bool; } /// The interval arm of the renewal loop. A `None` delay is a lease that never @@ -402,6 +409,12 @@ fn renewal_interval_for(expires_at: Option, now: Instant) -> RenewalDel #[async_trait] impl ShardManagerService for GrpcShardManagerService { + /// Yes: a real shard manager moves shards between executors, so two of them can believe they + /// own the same agent at once and only the storage can tell them apart. + fn requires_oplog_fencing(&self) -> bool { + true + } + async fn register( &self, port: u16, @@ -586,6 +599,12 @@ pub struct ShardManagerServiceSingleShard; #[async_trait] impl ShardManagerService for ShardManagerServiceSingleShard { + /// No: this executor owns the single shard for its whole life and nothing can take it away, + /// so there is no second writer to fence out. + fn requires_oplog_fencing(&self) -> bool { + false + } + async fn register( &self, _port: u16, @@ -874,6 +893,20 @@ mod tests { (service, shard_service) } + #[test] + // Which implementation is in effect is what decides whether the executor may start on an + // unfenced indexed storage, so both answers are pinned rather than left to the guard's caller. + fn only_the_real_shard_manager_requires_oplog_fencing() { + let mock = Arc::new(MockShardManager::new()); + let (service, _shard_service) = make_service(mock, Shutdown::new()); + + // Shards move between executors here, so two of them can believe they own the same agent. + assert!(service.requires_oplog_fencing()); + + // Nothing can take the single shard away, so there is no second writer to fence out. + assert!(!ShardManagerServiceSingleShard.requires_oplog_fencing()); + } + #[test] // The hook closes over the service graph that owns this service, so a strong reference here // would be a cycle nothing could free. `WorkerExecutorImpl` owns the hook; this only borrows diff --git a/golem-worker-executor/src/services/worker.rs b/golem-worker-executor/src/services/worker.rs index f10f540afc..ed69a37bdf 100644 --- a/golem-worker-executor/src/services/worker.rs +++ b/golem-worker-executor/src/services/worker.rs @@ -1782,7 +1782,7 @@ mod tests { use golem_common::model::regions::{DeletedRegions, OplogRegion}; use golem_common::model::{ AgentInvocationPayload, AgentInvocationResult, AgentMetadata, PendingInvocationRef, - PendingUpdateKind, PendingUpdateRef, ScanCursor, ShardLeaseRevision, + PendingUpdateKind, PendingUpdateRef, ScanCursor, ShardEpoch, ShardLeaseRevision, }; use golem_common::read_only_lock; use golem_service_base::model::component::Component; @@ -1850,6 +1850,7 @@ mod tests { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -1862,6 +1863,7 @@ mod tests { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -1874,6 +1876,7 @@ mod tests { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -2003,6 +2006,7 @@ mod tests { trace_states: Vec::new(), invocation_context: Vec::new(), wallet_pin: None, + shard_epoch: None, }, ); entries.insert( @@ -2840,6 +2844,7 @@ mod tests { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -2852,6 +2857,7 @@ mod tests { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -2864,6 +2870,7 @@ mod tests { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } diff --git a/golem-worker-executor/src/services/worker/session_index_tests.rs b/golem-worker-executor/src/services/worker/session_index_tests.rs index 4253016b76..6d127a0bb3 100644 --- a/golem-worker-executor/src/services/worker/session_index_tests.rs +++ b/golem-worker-executor/src/services/worker/session_index_tests.rs @@ -228,6 +228,7 @@ async fn create_oplog(service: &dyn OplogService, id: &OwnedAgentId) -> Arc Opl oplog .add(DurableStreamOplogRecord::Session(None, Box::new(record)).into_inline_entry()) .await + .expect("oplog write") } async fn append_noop(oplog: &dyn Oplog) -> OplogIndex { @@ -245,6 +247,7 @@ async fn append_noop(oplog: &dyn Oplog) -> OplogIndex { entity_parent_start_index: None, }) .await + .expect("oplog write") } async fn append_pending_invocation(oplog: &dyn Oplog, key: &IdempotencyKey) -> OplogIndex { @@ -257,6 +260,7 @@ async fn append_pending_invocation(oplog: &dyn Oplog, key: &IdempotencyKey) -> O Vec::new(), )) .await + .expect("oplog write") } fn attached_record( @@ -344,7 +348,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit }), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); storage.reset(); let metadata = service .lookup_durable_stream_control_metadata(&id, AgentMode::Durable, &key) @@ -401,7 +408,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit .finished .is_none() ); - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); storage.reset(); let metadata = reopened .lookup_durable_stream_control_metadata(&id, AgentMode::Durable, &key) @@ -446,7 +456,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit } } } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let status = AgentStatusRecord { oplog_idx: oplog.current_oplog_index().await, has_durable_stream_history: true, @@ -498,7 +511,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit active.push(prepared.attempt.session_key.clone()); append_session(oplog.as_ref(), record).await; } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let recovery = reopened .lookup_durable_stream_recovery_metadata(&id, AgentMode::Durable) .await @@ -563,7 +579,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit "raw preparation must be visible before commit" ); assert_eq!(cache.dirty, HashSet::from([raw_key])); - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); storage.reset(); cache .refresh( @@ -593,7 +612,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit ) .await; } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let recovery = reopened .lookup_durable_stream_recovery_metadata(&id, AgentMode::Durable) .await @@ -640,7 +662,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit ) .await; } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert!( restarted .lookup_durable_stream_recovery_metadata(&id, AgentMode::Durable) @@ -683,7 +708,10 @@ async fn persisted_control_projection_reopens_without_history_and_catches_commit .await; historical.push((attempt, offset)); } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!( restarted @@ -793,7 +821,10 @@ async fn closed_remote_consumer_streams_leave_recovery_across_epochs() { mappings.push(mapping); attachments.push(attachment); } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let mut cache = crate::worker::DurableTopologyRecoveryCache::default(); cache .refresh( @@ -861,7 +892,10 @@ async fn closed_remote_consumer_streams_leave_recovery_across_epochs() { cache.sessions.is_empty(), "raw closure must retire resident recovery work" ); - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert!( service .lookup_durable_stream_recovery_metadata(&owner, AgentMode::Durable) @@ -894,7 +928,10 @@ async fn closed_remote_consumer_streams_leave_recovery_across_epochs() { ) .await; } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); cache .refresh( oplog.as_ref(), @@ -1113,7 +1150,10 @@ async fn catchup_scans_multiple_chunks_and_recovers_evicted_completed_session() .await; bounded.insert(key, completed(prepared.as_u64(), finished.as_u64())); } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let horizon = oplog.current_oplog_index().await; assert!(horizon.as_u64() > 1024); assert!(bounded.iter().count() <= 128); @@ -1155,7 +1195,10 @@ async fn incremental_catchup_merges_later_fields_into_old_unfinished_session() { ) .await; } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let first_horizon = oplog.current_oplog_index().await; let status_at_first_horizon = AgentStatusRecord { oplog_idx: first_horizon, @@ -1199,7 +1242,10 @@ async fn incremental_catchup_merges_later_fields_into_old_unfinished_session() { }), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let mut later_status = status_at_first_horizon; later_status.oplog_idx = finished; @@ -1334,7 +1380,10 @@ async fn raw_attachment_authority_fences_before_commit_and_survives_buffer_drain }), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let producer = DurableStreamProducer::load( oplog.clone(), @@ -1379,7 +1428,10 @@ async fn raw_attachment_authority_fences_before_commit_and_survives_buffer_drain // No Worker/status actor participates: draining the buffer must not reset raw authority // back to the older published status used when this oplog was constructed. - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!( oplog_service.get_last_index(&id, AgentMode::Durable).await, resumed @@ -1435,7 +1487,10 @@ async fn raw_cold_reopen_ignores_stale_supplied_status_and_recovers_committed_re ), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let resumed_attempt = AttemptId::fresh(); let resumed = append_session( @@ -1458,7 +1513,10 @@ async fn raw_cold_reopen_ignores_stale_supplied_status_and_recovers_committed_re }), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); drop(oplog); let reopened = oplog_service @@ -1469,6 +1527,7 @@ async fn raw_cold_reopen_ignores_stale_supplied_status_and_recovers_committed_re agent_metadata(&id), stale_status(), suspended_status(), + None, ) .await; let raw = reopened @@ -1507,7 +1566,10 @@ async fn raw_cached_lookup_observes_takeover_committed_by_another_oplog_actor() ), ) .await; - first.commit(CommitLevel::Always).await; + first + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!( first .raw_durable_stream_session_status(&session_key) @@ -1527,6 +1589,7 @@ async fn raw_cached_lookup_observes_takeover_committed_by_another_oplog_actor() agent_metadata(&id), stale_status(), suspended_status(), + None, ) .await; let takeover_attempt = AttemptId::fresh(); @@ -1550,7 +1613,10 @@ async fn raw_cached_lookup_observes_takeover_committed_by_another_oplog_actor() }), ) .await; - second.commit(CommitLevel::Always).await; + second + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let observed = first .raw_durable_stream_session_status(&session_key) @@ -1591,7 +1657,10 @@ async fn raw_cache_eviction_recovers_finished_session_and_folds_buffered_then_co }), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert!( oplog .raw_durable_stream_session_status(&session_key) @@ -1627,7 +1696,10 @@ async fn raw_cache_eviction_recovers_finished_session_and_folds_buffered_then_co ); assert_eq!(buffered.attachment_attached, Some(false)); - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let committed = oplog .raw_durable_stream_session_status(&session_key) .await @@ -1663,7 +1735,10 @@ async fn persisted_exact_horizon_rejects_newer_index_and_offsets_hide_newer_atta ), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); service .stream_session_index .catch_up(&id, AgentMode::Durable, attached_idx) @@ -1749,7 +1824,10 @@ async fn raw_lookup_catches_up_archived_history_after_full_multilayer_reopen() { }), ) .await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert_eq!( MultiLayerOplog::try_archive_blocking(&oplog).await, Some(true) @@ -1796,6 +1874,7 @@ async fn raw_lookup_catches_up_archived_history_after_full_multilayer_reopen() { agent_metadata(&id), stale_status(), suspended_status(), + None, ) .await; let raw = reopened @@ -1851,7 +1930,10 @@ async fn indexed_raw_authority_cold_and_warm_lookups_do_not_read_oplog_history() for _ in 0..2048 { append_noop(oplog.as_ref()).await; } - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); let horizon = oplog.current_oplog_index().await; service .stream_session_index @@ -1867,6 +1949,7 @@ async fn indexed_raw_authority_cold_and_warm_lookups_do_not_read_oplog_history() agent_metadata(&id), stale_status(), suspended_status(), + None, ) .await; storage.reset(); @@ -1936,7 +2019,10 @@ async fn raw_authority_ignores_foreign_results_sharing_an_idempotency_key() { append_session(oplog.as_ref(), second.clone()).await; append_session(oplog.as_ref(), first).await; append_session(oplog.as_ref(), second).await; - oplog.commit(CommitLevel::Always).await; + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); assert!( oplog diff --git a/golem-worker-executor/src/services/worker_fork.rs b/golem-worker-executor/src/services/worker_fork.rs index 43c5e96160..737e1f4ca3 100644 --- a/golem-worker-executor/src/services/worker_fork.rs +++ b/golem-worker-executor/src/services/worker_fork.rs @@ -623,6 +623,10 @@ impl DefaultWorkerFork { timestamp: Timestamp::now_utc(), }, ))), + // Unfenced: the target's shard may belong to another executor, and + // this is a one-shot copy, not a live oplog. Its owner writes the metadata + // row on its first open. + None, ) .await; @@ -641,7 +645,7 @@ impl DefaultWorkerFork { &owned_source_agent_id.agent_id, &owned_target_agent_id.agent_id, ); - new_oplog.add(entry.clone()).await; + new_oplog.add(entry.clone()).await?; if let OplogEntry::Revert { dropped_region, .. } = &entry { deleted_regions_builder.add(dropped_region.clone()); @@ -700,7 +704,7 @@ impl DefaultWorkerFork { timestamp: now, idempotency_key, }) - .await; + .await?; } for target_revision in pending_update_revisions { @@ -713,7 +717,7 @@ impl DefaultWorkerFork { target_revision, details: Some("cancelled by fork".to_string()), }) - .await; + .await?; } Ok(new_oplog) @@ -839,7 +843,7 @@ impl WorkerForkService for DefaultWorkerFork { ) .await?; - new_oplog.commit(CommitLevel::Always).await; + new_oplog.commit(CommitLevel::Always).await?; // We go through worker proxy to resume the worker // as we need to make sure as it may live in another worker executor, @@ -916,7 +920,7 @@ impl WorkerForkService for DefaultWorkerFork { forced_commit: false, }), ) - .await; + .await?; if let Some(scope_start) = copied_scope_start { new_oplog @@ -926,10 +930,10 @@ impl WorkerForkService for DefaultWorkerFork { response: None, forced_commit: true, }) - .await; + .await?; } - new_oplog.commit(CommitLevel::Always).await; + new_oplog.commit(CommitLevel::Always).await?; // We go through worker proxy to resume the worker // as we need to make sure as it may live in another worker executor, @@ -991,6 +995,7 @@ mod tests { pinned_card_ids: Vec::new(), scope_card_id: None, }), + shard_epoch: None, }; match rewrite_forked_oplog_entry(entry, &source, &target) { diff --git a/golem-worker-executor/src/storage/indexed/memory.rs b/golem-worker-executor/src/storage/indexed/memory.rs index 6af19d2367..ea07055f4a 100644 --- a/golem-worker-executor/src/storage/indexed/memory.rs +++ b/golem-worker-executor/src/storage/indexed/memory.rs @@ -18,6 +18,7 @@ use crate::storage::indexed::{ }; use async_trait::async_trait; use golem_common::model::AgentId; +use golem_common::model::ShardEpoch; use regex::Regex; use std::collections::BTreeMap; use std::ops::Bound::Included; @@ -202,6 +203,7 @@ impl IndexedStorage for InMemoryIndexedStorage { key: &str, id: u64, value: Vec, + _shard_epoch: Option, ) -> Result<(), IndexedStorageError> { let primary_oplog_insert = matches!(&namespace, IndexedStorageNamespace::OpLog { .. }); let composite_key = Self::composite_key(namespace, key); @@ -389,6 +391,7 @@ mod tests { key, 1, &100, + None, ) .await .unwrap(); @@ -400,6 +403,7 @@ mod tests { key, 2, &200, + None, ) .await .unwrap(); @@ -411,6 +415,7 @@ mod tests { key, 3, &300, + None, ) .await .unwrap(); @@ -422,6 +427,7 @@ mod tests { key, 4, &400, + None, ) .await .unwrap(); @@ -455,6 +461,7 @@ mod tests { key, 1, &100, + None, ) .await .unwrap(); @@ -466,6 +473,7 @@ mod tests { key, 2, &200, + None, ) .await .unwrap(); @@ -477,6 +485,7 @@ mod tests { key, 3, &300, + None, ) .await .unwrap(); @@ -488,6 +497,7 @@ mod tests { key, 4, &400, + None, ) .await .unwrap(); @@ -521,6 +531,7 @@ mod tests { key, 10, &100, + None, ) .await .unwrap(); @@ -532,6 +543,7 @@ mod tests { key, 20, &200, + None, ) .await .unwrap(); @@ -543,6 +555,7 @@ mod tests { key, 30, &300, + None, ) .await .unwrap(); @@ -554,6 +567,7 @@ mod tests { key, 40, &400, + None, ) .await .unwrap(); @@ -587,6 +601,7 @@ mod tests { key, 10, &100, + None, ) .await .unwrap(); @@ -598,6 +613,7 @@ mod tests { key, 20, &200, + None, ) .await .unwrap(); @@ -609,6 +625,7 @@ mod tests { key, 30, &300, + None, ) .await .unwrap(); @@ -620,6 +637,7 @@ mod tests { key, 40, &400, + None, ) .await .unwrap(); @@ -654,6 +672,7 @@ mod tests { key, 10, &100, + None, ) .await .unwrap(); @@ -665,6 +684,7 @@ mod tests { key, 20, &200, + None, ) .await .unwrap(); @@ -676,6 +696,7 @@ mod tests { key, 30, &300, + None, ) .await .unwrap(); @@ -687,6 +708,7 @@ mod tests { key, 40, &400, + None, ) .await .unwrap(); @@ -721,6 +743,7 @@ mod tests { key, 10, &100, + None, ) .await .unwrap(); @@ -732,6 +755,7 @@ mod tests { key, 20, &200, + None, ) .await .unwrap(); @@ -764,6 +788,7 @@ mod tests { key, 10, &100, + None, ) .await .unwrap(); @@ -775,6 +800,7 @@ mod tests { key, 20, &200, + None, ) .await .unwrap(); @@ -807,6 +833,7 @@ mod tests { key, 1, &100, + None, ) .await .unwrap(); @@ -818,6 +845,7 @@ mod tests { key, 2, &200, + None, ) .await .unwrap(); @@ -829,6 +857,7 @@ mod tests { key, 3, &300, + None, ) .await .unwrap(); @@ -840,6 +869,7 @@ mod tests { key, 4, &400, + None, ) .await .unwrap(); diff --git a/golem-worker-executor/src/storage/indexed/mod.rs b/golem-worker-executor/src/storage/indexed/mod.rs index ddacff0eed..ccf1522a58 100644 --- a/golem-worker-executor/src/storage/indexed/mod.rs +++ b/golem-worker-executor/src/storage/indexed/mod.rs @@ -19,9 +19,10 @@ use std::time::Duration; use async_trait::async_trait; use bytes::Bytes; use desert_rust::{BinaryDeserializer, BinarySerializer}; -use golem_common::model::AgentId; use golem_common::model::agent::AgentMode; +use golem_common::model::{AgentId, ShardEpoch}; use golem_common::serialization::{deserialize, serialize}; +use golem_service_base::repo::RepoError; pub mod memory; pub mod multi_sqlite; @@ -45,6 +46,16 @@ pub enum IndexedStorageError { Conflict(String), /// Permanent error — data issue or schema error. Caller should not retry. Other(String), + /// The write was refused because the writer no longer owns the agent's shard: the epoch it + /// asserted is behind the one recorded for that oplog. + /// + /// Never retriable — retrying cannot make this executor the owner again. It is not a failure + /// of the storage either: the write was rejected on purpose, by a newer owner's claim. + Fenced { + key: String, + expected: ShardEpoch, + actual: Option, + }, } impl IndexedStorageError { @@ -62,6 +73,22 @@ impl Display for IndexedStorageError { } IndexedStorageError::Conflict(msg) => write!(f, "Storage conflict: {msg}"), IndexedStorageError::Other(msg) => write!(f, "Storage error: {msg}"), + IndexedStorageError::Fenced { + key, + expected, + actual, + } => match actual { + Some(actual) => write!( + f, + "Oplog write fenced for key {key}: asserted shard epoch {expected}, \ + the stored epoch is {actual}" + ), + None => write!( + f, + "Oplog write fenced for key {key}: asserted shard epoch {expected}, \ + but no epoch is stored for it" + ), + }, } } } @@ -74,6 +101,52 @@ impl From for IndexedStorageError { } } +/// Carries a fence rejection out of a transaction closure. +/// +/// [`Pool::with_tx_err`] requires its error type to be `From`, and +/// [`IndexedStorageError`] deliberately is not: each backend converts a `RepoError` through its +/// own classifier, which decides whether the failure is retriable and annotates a unique +/// violation as a possible ownership mismatch. A blanket `From` would flatten all of that into +/// `Other`. So the closure fails with this instead, and the backend maps it back at the boundary +/// with the classifier it would have used anyway - which keeps `with_tx_err`'s labelled rollback +/// and its metrics rather than hand-rolling `begin`/`rollback` at every early return. +#[derive(Debug)] +pub(crate) enum FencedTxError { + Repo(RepoError), + Fenced { + key: String, + expected: ShardEpoch, + actual: Option, + }, +} + +impl From for FencedTxError { + fn from(err: RepoError) -> Self { + FencedTxError::Repo(err) + } +} + +impl FencedTxError { + /// `classify` is the backend's own `RepoError` classifier. + pub(crate) fn into_indexed_storage_error( + self, + classify: fn(RepoError) -> IndexedStorageError, + ) -> IndexedStorageError { + match self { + FencedTxError::Repo(err) => classify(err), + FencedTxError::Fenced { + key, + expected, + actual, + } => IndexedStorageError::Fenced { + key, + expected, + actual, + }, + } + } +} + /// Generic indexed storage interface /// /// The storage holds indexes identified by keys. Each index is a sequence of entries, @@ -131,9 +204,16 @@ pub trait IndexedStorage: Debug + Sync { key: &str, id: u64, value: Vec, + shard_epoch: Option, ) -> Result<(), IndexedStorageError>; /// Appends multiple entries to the given key with the given id + /// + /// `shard_epoch` is the ownership generation the caller believes it holds for this key's + /// shard. A backend that fences checks it against the epoch recorded for the key, in the same + /// transaction as the insert, and refuses the whole batch with + /// [`IndexedStorageError::Fenced`] if it is behind. `None` asserts nothing and is for writers + /// that cannot know an epoch. The check is once per call, never per entry. async fn append_many( &self, svc_name: &'static str, @@ -142,6 +222,7 @@ pub trait IndexedStorage: Debug + Sync { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { for (id, value) in pairs.iter() { self.append( @@ -152,6 +233,7 @@ pub trait IndexedStorage: Debug + Sync { key, *id, value.to_vec(), + shard_epoch, ) .await?; } @@ -230,6 +312,49 @@ pub trait IndexedStorage: Debug + Sync { key: &str, last_dropped_id: u64, ) -> Result<(), IndexedStorageError>; + + /// Records the shard epoch that is authorised to write the given key, as a monotonic + /// compare-and-set: the write is accepted when `shard_epoch` is at least the stored one, and + /// refused with [`IndexedStorageError::Fenced`] when it is behind. Inserts the record if the + /// key has none. + /// + /// Monotonic rather than a plain overwrite so that a writer holding a stale epoch cannot walk + /// the record backwards and un-fence itself against the current owner. + /// + /// The default does nothing and accepts everything: a backend that cannot fence has no record + /// to keep. + async fn upsert_oplog_metadata( + &self, + _svc_name: &'static str, + _api_name: &'static str, + _namespace: IndexedStorageNamespace, + _key: &str, + _shard_epoch: ShardEpoch, + ) -> Result<(), IndexedStorageError> { + Ok(()) + } + + /// Forgets the epoch recorded for the given key. Called when the oplog itself is deleted, and + /// before its entries are, so that a writer still holding the old epoch is fenced by the + /// absent record rather than appending to an oplog that is being removed. + /// + /// Idempotent. The default does nothing. + async fn delete_oplog_metadata( + &self, + _svc_name: &'static str, + _api_name: &'static str, + _namespace: IndexedStorageNamespace, + _key: &str, + ) -> Result<(), IndexedStorageError> { + Ok(()) + } + + /// Whether this backend enforces `shard_epoch` on writes. Startup refuses a configuration + /// that pairs a backend answering `false` with a real shard manager, because the fence would + /// silently not exist. + fn supports_epoch_fencing(&self) -> bool { + false + } } pub trait IndexedStorageLabelledApi { @@ -389,6 +514,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { key: &str, id: u64, value: &V, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.storage .append( @@ -399,6 +525,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { key, id, serialize(value).map_err(IndexedStorageError::Other)?, + shard_epoch, ) .await } @@ -410,6 +537,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { key: &str, id: u64, value: Vec, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.storage .append( @@ -420,6 +548,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { key, id, value, + shard_epoch, ) .await } @@ -431,6 +560,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { namespace: &IndexedStorageNamespace, key: &str, pairs: &[(u64, &V)], + shard_epoch: Option, ) -> Result { let mut serialized_pairs = Vec::with_capacity(pairs.len()); let mut total_bytes = 0u64; @@ -439,7 +569,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { total_bytes += bytes.len() as u64; serialized_pairs.push((*id, Bytes::from(bytes))); } - self.append_many_raw(namespace, key, serialized_pairs.into()) + self.append_many_raw(namespace, key, serialized_pairs.into(), shard_epoch) .await?; Ok(total_bytes) } @@ -450,6 +580,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.storage .append_many( @@ -459,6 +590,7 @@ impl<'a, S: ?Sized + IndexedStorage> LabelledEntityIndexedStorage<'a, S> { namespace, key, pairs, + shard_epoch, ) .await } diff --git a/golem-worker-executor/src/storage/indexed/multi_sqlite.rs b/golem-worker-executor/src/storage/indexed/multi_sqlite.rs index 509ab2aa3f..ccd9f7b0dd 100644 --- a/golem-worker-executor/src/storage/indexed/multi_sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/multi_sqlite.rs @@ -22,6 +22,7 @@ use bytes::Bytes; use golem_common::cache::{BackgroundEvictionMode, Cache, FullCacheEvictionMode, SimpleCache}; use golem_common::config::DbSqliteConfig; use golem_common::model::AgentId; +use golem_common::model::ShardEpoch; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use std::path::{Path, PathBuf}; @@ -157,6 +158,40 @@ impl Debug for MultiSqliteIndexedStorage { #[async_trait] impl IndexedStorage for MultiSqliteIndexedStorage { + async fn upsert_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + shard_epoch: ShardEpoch, + ) -> Result<(), IndexedStorageError> { + self.storage_by_namespace(&namespace) + .await? + .upsert_oplog_metadata(svc_name, api_name, namespace, key, shard_epoch) + .await + } + + async fn delete_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + ) -> Result<(), IndexedStorageError> { + self.storage_by_namespace(&namespace) + .await? + .delete_oplog_metadata(svc_name, api_name, namespace, key) + .await + } + + /// Answers for the per-namespace databases this fans out to, which are all + /// [`SqliteIndexedStorage`]. Taking the trait default here would silently report "no fence" + /// for a backend that has one. + fn supports_epoch_fencing(&self) -> bool { + SqliteIndexedStorage::SUPPORTS_EPOCH_FENCING + } + async fn number_of_replicas( &self, _svc_name: &'static str, @@ -276,13 +311,27 @@ impl IndexedStorage for MultiSqliteIndexedStorage { key: &str, id: u64, value: Vec, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.storage_by_namespace(&namespace) .await? - .append(svc_name, api_name, entity_name, namespace, key, id, value) + .append( + svc_name, + api_name, + entity_name, + namespace, + key, + id, + value, + shard_epoch, + ) .await } + /// Overridden rather than inherited. The trait default loops [`Self::append`], which would + /// resolve the per-agent database and re-check the fence once per entry, in a separate + /// transaction each time - so a batch could land half-written, and the contract that the + /// fence is checked once per call would not hold. async fn append_many( &self, svc_name: &'static str, @@ -291,10 +340,19 @@ impl IndexedStorage for MultiSqliteIndexedStorage { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { self.storage_by_namespace(namespace) .await? - .append_many(svc_name, api_name, entity_name, namespace, key, pairs) + .append_many( + svc_name, + api_name, + entity_name, + namespace, + key, + pairs, + shard_epoch, + ) .await } @@ -438,6 +496,7 @@ mod tests { &first_namespace, "shared-key", vec![(1, Bytes::from_static(b"first-agent-value"))].into(), + None, ) .await .unwrap(); @@ -449,6 +508,7 @@ mod tests { &second_namespace, "shared-key", vec![(1, Bytes::from_static(b"second-agent-value"))].into(), + None, ) .await .unwrap(); diff --git a/golem-worker-executor/src/storage/indexed/postgres.rs b/golem-worker-executor/src/storage/indexed/postgres.rs index 6a75ee811b..091022cab8 100644 --- a/golem-worker-executor/src/storage/indexed/postgres.rs +++ b/golem-worker-executor/src/storage/indexed/postgres.rs @@ -13,8 +13,8 @@ // limitations under the License. use super::{ - IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, IndexedStorageNamespace, - ScanCursor, + FencedTxError, IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, + IndexedStorageNamespace, ScanCursor, }; use crate::services::golem_config::IndexedStoragePostgresConfig; use async_trait::async_trait; @@ -22,6 +22,7 @@ use bytes::Bytes; use futures::FutureExt; use golem_common::SafeDisplay; use golem_common::metrics::db::record_db_serialized_size; +use golem_common::model::ShardEpoch; use golem_service_base::db::postgres::PostgresPool; use golem_service_base::db::{Pool, PoolApi}; use golem_service_base::migration::{IncludedMigrationsDir, Migrations}; @@ -156,6 +157,12 @@ impl PostgresIndexedStorage { Self::classify_repo_error(err, false) } + /// The oplog-insert classifier as a plain `fn`, so it can be handed to + /// [`FencedTxError::into_indexed_storage_error`], which takes a function pointer. + fn classify_repo_error_oplog_insert(err: RepoError) -> IndexedStorageError { + Self::classify_repo_error(err, true) + } + async fn acquire_permit(&self) -> Option { match &self.semaphore { Some(sem) => Some(sem.clone().acquire_owned().await.expect("semaphore closed")), @@ -273,6 +280,9 @@ impl IndexedStorage for PostgresIndexedStorage { Ok((new_cursor, keys)) } + /// Delegates to [`Self::append_many`] so there is exactly one fenced write path: the epoch + /// check has to happen in the same transaction as the insert, and a lone `INSERT` is not in + /// one. The permit is acquired there, not here. async fn append( &self, svc_name: &'static str, @@ -282,25 +292,18 @@ impl IndexedStorage for PostgresIndexedStorage { key: &str, id: u64, value: Vec, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { - let _permit = self.acquire_permit().await; - record_db_serialized_size(DB_TYPE, svc_name, entity_name, value.len()); - let primary_oplog_insert = matches!(&namespace, IndexedStorageNamespace::OpLog { .. }); - let id = Self::to_i64(id, "id")?; - let query = sqlx::query( - "INSERT INTO index_storage (namespace, key, id, value) VALUES ($1, $2, $3, $4);", + self.append_many( + svc_name, + api_name, + entity_name, + &namespace, + key, + vec![(id, Bytes::from(value))].into(), + shard_epoch, ) - .bind(Self::namespace(namespace)) - .bind(key) - .bind(id) - .bind(value); - - self.pool - .with_rw(svc_name, api_name) - .execute(query) - .await - .map(|_| ()) - .map_err(|err| Self::classify_repo_error(err, primary_oplog_insert)) + .await } async fn append_many( @@ -311,24 +314,11 @@ impl IndexedStorage for PostgresIndexedStorage { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { if pairs.is_empty() { return Ok(()); } - if let [(id, value)] = pairs.as_ref() { - return self - .append( - svc_name, - api_name, - entity_name, - (*namespace).clone(), - key, - *id, - value.to_vec(), - ) - .await; - } - let _permit = self.acquire_permit().await; let primary_oplog_insert = matches!(namespace, IndexedStorageNamespace::OpLog { .. }); let namespace = Self::namespace((*namespace).clone()); @@ -339,8 +329,37 @@ impl IndexedStorage for PostgresIndexedStorage { } self.pool - .with_tx(svc_name, api_name, |tx| { + .with_tx_err::<(), FencedTxError, _>(svc_name, api_name, |tx| { async move { + + // Inside the insert transaction, and holding the row, so a writer that has + // lost the shard cannot slip a batch in between the check and the insert. + // `FOR UPDATE` is what serialises two executors racing over the same oplog. + if let Some(expected) = shard_epoch { + let stored: Option<(i64,)> = tx + .fetch_optional_as( + sqlx::query_as( + "SELECT epoch FROM oplog_metadata WHERE namespace = $1 AND key = $2 FOR UPDATE;", + ) + .bind(namespace.clone()) + .bind(key.clone()), + ) + .await?; + let actual = stored.map(|(epoch,)| ShardEpoch(epoch as u64)); + // Strict equality: the monotonic rule belongs to the upsert. A stored + // epoch above ours means a newer owner has taken over; below ours means + // an open skipped the assertion. Neither is ours to write through. An + // absent row fences too - it is written before the first entry and + // removed before the last. + if actual != Some(expected) { + return Err(FencedTxError::Fenced { + key: key.clone(), + expected, + actual, + }); + } + } + for chunk in pairs.chunks(Self::APPEND_MANY_CHUNK_SIZE) { let mut query_builder = QueryBuilder::::new( "INSERT INTO index_storage (namespace, key, id, value) ", @@ -363,7 +382,92 @@ impl IndexedStorage for PostgresIndexedStorage { .boxed() }) .await - .map_err(|err| Self::classify_repo_error(err, primary_oplog_insert)) + .map_err(|err| { + err.into_indexed_storage_error(if primary_oplog_insert { + Self::classify_repo_error_oplog_insert + } else { + Self::classify_repo_error_general + }) + }) + } + + /// Monotonic compare-and-set on the epoch authorised to write this key. + /// + /// The `WHERE` on the conflict path is what makes it monotonic: a lower epoch updates no row, + /// so a writer holding a stale epoch cannot walk the record back and un-fence itself against + /// the current owner. Postgres reports one row affected for an insert and for an accepted + /// update, and zero when the `WHERE` excludes it. + async fn upsert_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + shard_epoch: ShardEpoch, + ) -> Result<(), IndexedStorageError> { + let _permit = self.acquire_permit().await; + let namespace = Self::namespace(namespace); + let epoch = Self::to_i64(shard_epoch.0, "shard_epoch")?; + + let mut api = self.pool.with_rw(svc_name, api_name); + let result = api + .execute( + sqlx::query( + r#"INSERT INTO oplog_metadata (namespace, key, epoch) VALUES ($1, $2, $3) + ON CONFLICT (namespace, key) DO UPDATE SET epoch = EXCLUDED.epoch + WHERE oplog_metadata.epoch <= EXCLUDED.epoch;"#, + ) + .bind(namespace.clone()) + .bind(key) + .bind(epoch), + ) + .await + .map_err(Self::classify_repo_error_general)?; + + if result.rows_affected() == 0 { + // Rejected. Read the stored epoch back purely so the error can name it. + let stored: Option<(i64,)> = api + .fetch_optional_as( + sqlx::query_as( + "SELECT epoch FROM oplog_metadata WHERE namespace = $1 AND key = $2;", + ) + .bind(namespace) + .bind(key), + ) + .await + .map_err(Self::classify_repo_error_general)?; + return Err(IndexedStorageError::Fenced { + key: key.to_string(), + expected: shard_epoch, + actual: stored.map(|(epoch,)| ShardEpoch(epoch as u64)), + }); + } + + Ok(()) + } + + async fn delete_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + ) -> Result<(), IndexedStorageError> { + let _permit = self.acquire_permit().await; + let query = sqlx::query("DELETE FROM oplog_metadata WHERE namespace = $1 AND key = $2;") + .bind(Self::namespace(namespace)) + .bind(key); + + self.pool + .with_rw(svc_name, api_name) + .execute(query) + .await + .map(|_| ()) + .map_err(Self::classify_repo_error_general) + } + + fn supports_epoch_fencing(&self) -> bool { + true } async fn length( diff --git a/golem-worker-executor/src/storage/indexed/redis.rs b/golem-worker-executor/src/storage/indexed/redis.rs index d16a31879d..fb27600088 100644 --- a/golem-worker-executor/src/storage/indexed/redis.rs +++ b/golem-worker-executor/src/storage/indexed/redis.rs @@ -23,6 +23,7 @@ use fred::prelude::{Key, Value}; use fred::types::config::Options; use fred::types::streams::XCapKind; use golem_common::metrics::redis::{record_redis_deserialized_size, record_redis_serialized_size}; +use golem_common::model::ShardEpoch; use golem_common::redis::{RedisError, RedisPool}; use std::collections::HashMap; use std::sync::Arc; @@ -245,6 +246,7 @@ impl IndexedStorage for RedisIndexedStorage { key: &str, id: u64, value: Vec, + _shard_epoch: Option, ) -> Result<(), IndexedStorageError> { record_redis_serialized_size(svc_name, entity_name, value.len()); let primary_oplog_insert = matches!(&namespace, IndexedStorageNamespace::OpLog { .. }); @@ -277,6 +279,7 @@ impl IndexedStorage for RedisIndexedStorage { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + _shard_epoch: Option, ) -> Result<(), IndexedStorageError> { if !pairs.is_empty() { let primary_oplog_insert = matches!(namespace, IndexedStorageNamespace::OpLog { .. }); diff --git a/golem-worker-executor/src/storage/indexed/sqlite.rs b/golem-worker-executor/src/storage/indexed/sqlite.rs index 8e815dc3d1..1fcf4f145e 100644 --- a/golem-worker-executor/src/storage/indexed/sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/sqlite.rs @@ -13,15 +13,15 @@ // limitations under the License. use super::{ - IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, IndexedStorageNamespace, - ScanCursor, + FencedTxError, IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, + IndexedStorageNamespace, ScanCursor, }; use async_trait::async_trait; use bytes::Bytes; -use futures::FutureExt; use golem_common::SafeDisplay; use golem_common::config::DbSqliteConfig; use golem_common::metrics::db::record_db_serialized_size; +use golem_common::model::ShardEpoch; use golem_service_base::db::sqlite::SqlitePool; use golem_service_base::db::{Pool, PoolApi}; use golem_service_base::migration::{IncludedMigrationsDir, Migrations}; @@ -40,6 +40,14 @@ pub struct SqliteIndexedStorage { } impl SqliteIndexedStorage { + /// Whether this backend enforces the shard-epoch fence on writes. + /// + /// A constant rather than a literal in the trait impl because + /// [`super::multi_sqlite::MultiSqliteIndexedStorage`] is a fan-out of these and must always + /// answer the same way: it has no namespace to delegate the question through, so this is what + /// keeps the two from drifting apart. + pub(crate) const SUPPORTS_EPOCH_FENCING: bool = true; + pub async fn configured(config: &DbSqliteConfig) -> Result { Self::migrate(config).await?; @@ -137,6 +145,10 @@ impl SqliteIndexedStorage { #[async_trait] impl IndexedStorage for SqliteIndexedStorage { + fn supports_epoch_fencing(&self) -> bool { + Self::SUPPORTS_EPOCH_FENCING + } + async fn number_of_replicas( &self, _svc_name: &'static str, @@ -221,6 +233,8 @@ impl IndexedStorage for SqliteIndexedStorage { Ok((new_cursor, keys)) } + /// Delegates to [`Self::append_many`] so there is exactly one fenced write path: the epoch + /// check has to happen in the same transaction as the insert. async fn append( &self, svc_name: &'static str, @@ -230,31 +244,18 @@ impl IndexedStorage for SqliteIndexedStorage { key: &str, id: u64, value: Vec, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { - record_db_serialized_size(DB_TYPE, svc_name, entity_name, value.len()); - let primary_oplog_insert = matches!(&namespace, IndexedStorageNamespace::OpLog { .. }); - let query = sqlx::query( - r#" - INSERT INTO index_storage (namespace, key, id, value) VALUES (?,?,?,?); - "#, + self.append_many( + svc_name, + api_name, + entity_name, + &namespace, + key, + vec![(id, Bytes::from(value))].into(), + shard_epoch, ) - .bind(Self::namespace(namespace)) - .bind(key) - .bind(sqlx::types::Json(id)) - .bind(value); - - self.pool - .with_rw(svc_name, api_name) - .execute(query) - .await - .map(|_| ()) - .map_err(|err| { - if primary_oplog_insert { - Self::classify_repo_error_primary_oplog_insert(err) - } else { - Self::classify_repo_error(err) - } - }) + .await } async fn append_many( @@ -265,6 +266,7 @@ impl IndexedStorage for SqliteIndexedStorage { namespace: &IndexedStorageNamespace, key: &str, pairs: Arc<[(u64, Bytes)]>, + shard_epoch: Option, ) -> Result<(), IndexedStorageError> { if pairs.is_empty() { return Ok(()); @@ -278,8 +280,33 @@ impl IndexedStorage for SqliteIndexedStorage { } self.pool - .with_tx(svc_name, api_name, |tx| { - async move { + .with_tx_err::<(), FencedTxError, _>(svc_name, api_name, |tx| { + Box::pin(async move { + // SQLite has no `SELECT ... FOR UPDATE`, and it does not need one here: the + // write pool is capped at a single connection (golem-service-base + // db/sqlite.rs:46-50), so this transaction holds the only writer and the + // check cannot be interleaved. Raising that cap means switching this to + // `BEGIN IMMEDIATE`. + if let Some(expected) = shard_epoch { + let stored: Option<(i64,)> = tx + .fetch_optional_as( + sqlx::query_as( + "SELECT epoch FROM oplog_metadata WHERE namespace = ? AND key = ?;", + ) + .bind(namespace.clone()) + .bind(key.clone()), + ) + .await?; + let actual = stored.map(|(epoch,)| ShardEpoch(epoch as u64)); + if actual != Some(expected) { + return Err(FencedTxError::Fenced { + key: key.clone(), + expected, + actual, + }); + } + } + for (id, value) in pairs.iter() { tx.execute( sqlx::query( @@ -294,19 +321,90 @@ impl IndexedStorage for SqliteIndexedStorage { } Ok(()) - } - .boxed() + }) }) .await .map_err(|err| { - if primary_oplog_insert { - Self::classify_repo_error_primary_oplog_insert(err) + err.into_indexed_storage_error(if primary_oplog_insert { + Self::classify_repo_error_primary_oplog_insert } else { - Self::classify_repo_error(err) - } + Self::classify_repo_error + }) }) } + /// Monotonic compare-and-set: the `WHERE` on the conflict path means a lower epoch updates no + /// row, so a stale writer cannot walk the record back and un-fence itself. The unqualified + /// `epoch` there is the existing row's. + async fn upsert_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + shard_epoch: ShardEpoch, + ) -> Result<(), IndexedStorageError> { + let namespace = Self::namespace(namespace); + // `i64`, not `u64`: sqlx has no `Encode` for `u64`, which is why ids elsewhere in + // this file go through `Json`. That encodes as TEXT, and comparison affinity is applied + // per operand, so this column stays integer-bound everywhere. + let epoch = shard_epoch.0 as i64; + + let mut api = self.pool.with_rw(svc_name, api_name); + let result = api + .execute( + sqlx::query( + r#"INSERT INTO oplog_metadata (namespace, key, epoch) VALUES (?, ?, ?) + ON CONFLICT(namespace, key) DO UPDATE SET epoch = excluded.epoch + WHERE epoch <= excluded.epoch;"#, + ) + .bind(namespace.clone()) + .bind(key) + .bind(epoch), + ) + .await + .map_err(Self::classify_repo_error)?; + + if result.rows_affected() == 0 { + let stored: Option<(i64,)> = api + .fetch_optional_as( + sqlx::query_as( + "SELECT epoch FROM oplog_metadata WHERE namespace = ? AND key = ?;", + ) + .bind(namespace) + .bind(key), + ) + .await + .map_err(Self::classify_repo_error)?; + return Err(IndexedStorageError::Fenced { + key: key.to_string(), + expected: shard_epoch, + actual: stored.map(|(epoch,)| ShardEpoch(epoch as u64)), + }); + } + + Ok(()) + } + + async fn delete_oplog_metadata( + &self, + svc_name: &'static str, + api_name: &'static str, + namespace: IndexedStorageNamespace, + key: &str, + ) -> Result<(), IndexedStorageError> { + let query = sqlx::query("DELETE FROM oplog_metadata WHERE namespace = ? AND key = ?;") + .bind(Self::namespace(namespace)) + .bind(key); + + self.pool + .with_rw(svc_name, api_name) + .execute(query) + .await + .map(|_| ()) + .map_err(Self::classify_repo_error) + } + async fn length( &self, svc_name: &'static str, @@ -529,6 +627,7 @@ mod tests { (2, Bytes::from_static(b"second")), ] .into(), + None, ) .await .unwrap(); @@ -566,6 +665,7 @@ mod tests { "oplog", 2, b"existing".to_vec(), + None, ) .await .unwrap(); @@ -582,6 +682,7 @@ mod tests { (2, Bytes::from_static(b"conflict")), ] .into(), + None, ) .await; diff --git a/golem-worker-executor/src/worker/instance.rs b/golem-worker-executor/src/worker/instance.rs index dd3e3688a3..9e7692580e 100644 --- a/golem-worker-executor/src/worker/instance.rs +++ b/golem-worker-executor/src/worker/instance.rs @@ -21,7 +21,7 @@ use crate::durable_host::tool::operation::{DeferredAdmissionTable, OwnerToolOper use crate::model::ExecutionStatus; use crate::services::active_agents::WorkerComponentCharge; use crate::services::agent_filesystem::FilesystemGenerationHandle; -use crate::services::oplog::{CommitLevel, Oplog}; +use crate::services::oplog::{CommitLevel, Oplog, OplogError}; use crate::services::resource_limits::AtomicResourceEntry; use crate::services::{HasActiveAgents, HasComponentService, HasWasmtimeEngine}; use crate::workerctx::WorkerCtx; @@ -354,7 +354,16 @@ impl OwnerExecution { pub(crate) async fn test_after_monotonic_clock_start(&self) -> Result<(), InterruptKind> { let gate = self.monotonic_clock_start_gate.lock().unwrap().take(); if let Some(gate) = gate { - self.oplog.commit(CommitLevel::Always).await; + // The gate is entered from inside a host call, where the fence surfaces as the + // interrupt that gives the agent up; a transient storage failure is fatal here as + // everywhere else. + match self.oplog.commit(CommitLevel::Always).await { + Ok(_) => {} + Err(OplogError::Fenced(_)) => { + return Err(InterruptKind::ShardLost); + } + Err(error) => panic!("oplog write: {error}"), + } if let Some(entered) = gate.entered.lock().unwrap().take() { let _ = entered.send(()); } @@ -414,12 +423,6 @@ impl OwnerExecution { pub async fn commit(&self, level: CommitLevel) -> OplogIndex { self.commit.commit_and_update_state(level).await.0 } - - pub async fn add_and_commit(&self, entry: OplogEntry) -> OplogIndex { - let index = self.oplog.add(entry).await; - self.commit(CommitLevel::Always).await; - index - } } /// Owner-scoped runtime resources reused by primary and entity Store construction. diff --git a/golem-worker-executor/src/worker/invocation.rs b/golem-worker-executor/src/worker/invocation.rs index c87cb201f6..a0b4db5f8c 100644 --- a/golem-worker-executor/src/worker/invocation.rs +++ b/golem-worker-executor/src/worker/invocation.rs @@ -114,6 +114,13 @@ pub async fn invoke_observed_and_traced( record_invocation(was_live_before, "suspended"); result } + Ok(InvokeResult::Interrupted { + interrupt_kind: InterruptKind::ShardLost, + .. + }) => { + record_invocation(was_live_before, "shard_lost"); + result + } Ok(InvokeResult::Interrupted { .. }) => { record_invocation(was_live_before, "restarted"); result diff --git a/golem-worker-executor/src/worker/invocation_loop.rs b/golem-worker-executor/src/worker/invocation_loop.rs index 5409668005..3123337515 100644 --- a/golem-worker-executor/src/worker/invocation_loop.rs +++ b/golem-worker-executor/src/worker/invocation_loop.rs @@ -313,6 +313,16 @@ impl InvocationLoop { self.stop_unloaded(None).await; break; } + InterruptKind::ShardLost => { + // Nothing is written: the oplog belongs to the shard's new owner + // now. Whoever was waiting for this start is told to look there. + self.parent.complete_startup( + self.start_attempt, + Err(WorkerExecutorError::ShardingNotReady), + ); + self.stop_unloaded(None).await; + break; + } } } CreateInstanceResult::Failed => { @@ -664,6 +674,8 @@ impl InvocationLoop { self.parent.add_and_commit_oplog(OplogEntry::interrupted()).await; } InterruptKind::Restart | InterruptKind::Jump => {} + // The oplog is the new owner's to write. + InterruptKind::ShardLost => {} } if matches!(kind, InterruptKind::Interrupt(_)) && let Some(key) = current_idempotency_key @@ -2382,6 +2394,24 @@ impl Invocation<'_, Ctx> { } failed_agent_invocation_outcome(self.parent.agent_mode(), decision) } + // Intercepted before the arm below, which would flatten it into an + // `AgentError::InternalError` and append an `Error` entry to the very oplog that + // just refused the write. + Err(error @ WorkerExecutorError::OplogFenced { .. }) => { + let decision = self + .store + .data_mut() + .on_invocation_failure( + &full_function_name, + &TrapType::Interrupt(InterruptKind::ShardLost), + ) + .await; + let _ = self + .parent + .fail_durable_streaming_session(idempotency_key, error.to_string()) + .await; + failed_agent_invocation_outcome(self.parent.agent_mode(), decision) + } Err(error) => { self.store .data_mut() diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index e02b6065d1..beae704bcc 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -70,7 +70,9 @@ use crate::services::events::{Event, EventsSubscription}; use crate::services::golem_config::SnapshotPolicy; use crate::services::linear_memory::{LinearMemoryTracker, SHARED_LINEAR_MEMORY_ERROR}; use crate::services::oplog::plugin::ForwardingOplog; -use crate::services::oplog::{CommitLevel, Oplog, OplogOps, downcast_oplog}; +use crate::services::oplog::{ + CommitLevel, Oplog, OplogError, OplogFence, OplogOps, downcast_oplog, +}; use crate::services::resource_limits::AtomicResourceEntry; use crate::services::resource_usage_metering::ResourceUsageAccount; use crate::services::worker::{ @@ -132,7 +134,7 @@ use golem_common::model::worker::{ use golem_common::model::{ AgentFingerprint, AgentId, AgentInvocation, AgentInvocationOutput, AgentInvocationPayload, AgentInvocationResult, AgentMetadata, AgentStatusRecord, IdempotencyKey, OwnedAgentId, - PendingInvocationRef, PendingUpdateKind, PendingUpdateRef, Timestamp, + PendingInvocationRef, PendingUpdateKind, PendingUpdateRef, ShardEpoch, ShardId, Timestamp, TimestampedAgentInvocation, }; use golem_common::one_shot::OneShotEvent; @@ -481,6 +483,9 @@ pub struct Worker { /// Prevents weak-reference background work from starting while an unloaded /// worker is being conditionally removed from `ActiveAgents`. cache_retirement_in_progress: AtomicBool, + /// Set once this executor has given the agent up. One-shot: the first reason wins, and the + /// agent is never revived here. + relinquishment: std::sync::OnceLock, startup_attempt: StartupAttemptTracker, linear_memory_grant: StdMutex>>>, /// Lifecycle request shared across resident worker generations. A terminal request is retained @@ -730,8 +735,65 @@ impl Worker { .unwrap_or_else(|| "-".to_string()) } + /// Records that this executor is giving the agent up. Idempotent; the first reason wins. + /// + /// Synchronous and lock-free on purpose: the stop path calls it while holding the instance + /// lock, where anything that could take that lock again would deadlock. + pub(crate) fn mark_relinquished(&self, reason: RelinquishReason) -> bool { + self.relinquishment.set(reason).is_ok() + } + + pub(crate) fn is_relinquished(&self) -> bool { + self.relinquishment.get().is_some() + } + + /// What anyone waiting on this agent is told. Every variant is one the worker service answers + /// by refreshing its routing table and retrying, so the invocation lands on the new owner + /// rather than failing. + pub(crate) fn relinquish_error(&self) -> WorkerExecutorError { + self.relinquishment + .get() + .map_or(WorkerExecutorError::ShardingNotReady, |reason| { + reason.to_error() + }) + } + + /// Give the agent up: stop it here without writing to its oplog or its status, and drop it + /// from this executor so the worker service resumes it on the shard's owner. + /// + /// Never a restart in place - that would reopen the oplog with the same stale epoch and let + /// this executor keep writing to an agent it no longer owns. + pub(crate) async fn relinquish(&self, reason: RelinquishReason) { + self.mark_relinquished(reason); + let error = self.relinquish_error(); + // Signalled before the stop so a running guest actually leaves wasmtime; the loop then + // exits through `stop_internal`, which is where the agent is dropped. The ack is + // deliberately not awaited: a caller that blocks on it would panic if the worker was + // already stopping and its broadcast sender had gone. + self.set_interrupting_for(InterruptKind::ShardLost, UnloadReason::ShardLost) + .await; + self.stop_internal( + false, + Some(error.clone()), + UnloadRequest::ordinary(UnloadReason::ShardLost), + FinalWorkerState::Unloaded { + startup_failure: Some(error), + }, + PendingLiveInvocationDisposition::Fail, + ) + .await; + } + pub(crate) async fn remove_from_active_agents(&self) { - self.deps.active_agents().remove(&self.owned_agent_id).await; + match self.relinquishment.get() { + Some(reason) => { + self.deps + .active_agents() + .remove_with(&self.owned_agent_id, reason.owner_failure()) + .await + } + None => self.deps.active_agents().remove(&self.owned_agent_id).await, + } } /// Gets or creates a worker, but does not start it @@ -1098,6 +1160,7 @@ impl Worker { }), instance, cache_retirement_in_progress: AtomicBool::new(false), + relinquishment: std::sync::OnceLock::new(), startup_attempt: StartupAttemptTracker::default(), linear_memory_grant: StdMutex::new(None), interrupt_signal: Arc::new(async_lock::Mutex::new(WorkerInterruptState::default())), @@ -3142,7 +3205,8 @@ impl Worker { loop { let delta = growth.delta.swap(0, Ordering::AcqRel); if delta > 0 { - self.add_to_oplog(OplogEntry::grow_memory(delta)).await; + self.add_to_oplog_or_relinquish(OplogEntry::grow_memory(delta)) + .await; } let current_growth = self.memory_growth.lock().unwrap(); @@ -3417,7 +3481,7 @@ impl Worker { status.invocation_results.revert_generation(), instance_guard, ) - .await + .await? { continue; } @@ -3944,7 +4008,7 @@ impl Worker { ) }), ) - .await; + .await?; streams .commit_consumer_journal() .await @@ -5098,10 +5162,34 @@ impl Worker { /// Appends an oplog entry without forcing a durable commit. Callers that /// require ordering must await the append before exposing subsequent work. - pub async fn add_to_oplog(&self, entry: OplogEntry) -> OplogIndex { + pub async fn add_to_oplog(&self, entry: OplogEntry) -> Result { self.oplog.add(entry).await } + /// Appends an entry on a path that has no way to report the failure to its caller. + /// + /// A fenced write means the shard moved while this agent was resident. The agent is marked + /// relinquished so the stop that follows drops it from this executor rather than writing to an + /// oplog another executor owns now, and `OplogIndex::NONE` is returned for the entry that was + /// not written - the same "no index" value a debugging session's discarded write returns. + /// + /// Marked rather than stopped here on purpose: these callers run under the instance lock and + /// inside the wasm store, where `relinquish` would deadlock on the lock it already holds. The + /// fence latches on the oplog, so the invocation's next write is refused too and unwinds the + /// loop, which is where the stop belongs. + /// + /// Every other storage failure keeps the fail-stop behaviour it has always had. + pub async fn add_to_oplog_or_relinquish(&self, entry: OplogEntry) -> OplogIndex { + match self.oplog.add(entry).await { + Ok(index) => index, + Err(OplogError::Fenced(fence)) => { + self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence)))); + OplogIndex::NONE + } + Err(error) => panic!("oplog write: {error}"), + } + } + pub async fn commit_oplog_and_update_state(&self, commit_level: CommitLevel) -> OplogIndex { let (result, changed) = self.state_actor.commit_and_update_state(commit_level).await; if changed { @@ -5117,7 +5205,7 @@ impl Worker { // Should only be called from invocation loop pub async fn add_and_commit_oplog(&self, entry: OplogEntry) -> OplogIndex { - let result = self.add_to_oplog(entry).await; + let result = self.add_to_oplog_or_relinquish(entry).await; self.commit_oplog_and_update_state(CommitLevel::Always) .await; result @@ -5164,7 +5252,7 @@ impl Worker { let mut queued_event_indices = Vec::with_capacity(card_ids.len()); for card_id in card_ids { queued_event_indices.push( - self.add_to_oplog(OplogEntry::card_event_queued( + self.add_to_oplog_or_relinquish(OplogEntry::card_event_queued( None, QueuedCardEvent::revoke(card_id), )) @@ -5254,7 +5342,7 @@ impl Worker { entry: OplogEntry, wakeup: Option, ) -> OplogIndex { - let result = self.add_to_oplog(entry).await; + let result = self.add_to_oplog_or_relinquish(entry).await; // The caller already holds the instance lock (and sends the wakeup itself below), so // this must not enqueue a `NotifyStatusChanged` lifecycle job: the commit job is safe to // await while holding the instance lock precisely because the status task never takes @@ -5737,6 +5825,14 @@ impl Worker { drop(instance_guard); self.handle_stop_result(stop_result).await; + + // The single removal point. Every loop exit and every external stop passes through here, + // so a relinquished agent is dropped from this executor exactly once - and only after the + // loop has actually gone, so the new owner cannot recover it while it is still running. + if self.is_relinquished() { + self.remove_from_active_agents().await; + } + if !called_from_invocation_loop && let Some(startup_attempt) = startup_attempt { self.complete_startup(startup_attempt, Err(startup_error)); } @@ -5860,18 +5956,36 @@ impl Worker { self.fail_pending_invocations(error.clone()).await; }; - // Make sure the oplog is committed - self.oplog.commit(CommitLevel::Always).await; + // Make sure the oplog is committed. Best-effort: a stop must finish. + let fenced = match self.oplog.commit(CommitLevel::Always).await { + Ok(_) => false, + Err(OplogError::Fenced(fence)) => { + // The shard has a new owner. `mark_relinquished` is synchronous and takes + // no lock, so it is safe under the instance lock this arm holds - calling + // `relinquish` here would deadlock on that same lock. + self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence)))); + true + } + Err(error) => { + warn!("Committing the oplog while stopping failed: {error}"); + false + } + }; // Persist any pending cached-status changes synchronously before the worker leaves // memory, so a subsequent cold load does not have to re-fold oplog entries that were // only reflected in the (deferred) in-memory status. Best-effort: a failure is // logged/metered inside `flush` and re-queued; the blob is reconstructable from the // oplog, so it must not block the stop. - if let Err(err) = self - .status_flusher - .flush(status_flusher::FlushReason::Forced) - .await + // + // Skipped entirely when the commit was fenced: this is a key-value write, which is + // NOT fenced, so it would happily overwrite the new owner's newer status blob with + // our stale one. + if !fenced + && let Err(err) = self + .status_flusher + .flush(status_flusher::FlushReason::Forced) + .await { debug!("Forced status flush on stop failed (will retry in background): {err}"); } @@ -6178,6 +6292,7 @@ impl Worker { + HasConfig + HasOplogService + HasEnvironmentStateService + + HasShardService + Sync, >( this: &T, @@ -6188,6 +6303,11 @@ impl Worker { parent: Option, freshness_disposition: InvocationFreshnessDisposition, ) -> Result { + // Captured once, here, and cached for the life of the oplog. One live oplog is one + // ownership generation: a renewal never moves an epoch, and when one does move this + // executor is the side that lost the shard, so re-reading it per write would only let a + // losing executor talk itself back into ownership. + let shard_epoch = owned_shard_epoch(this, &owned_agent_id.agent_id); let component_id = owned_agent_id.component_id(); // KnownFresh has already been validated against the ephemeral agent type, phantom ID, and @@ -6274,6 +6394,7 @@ impl Worker { initial_worker_metadata.clone(), read_only_lock::arc_swap::ReadOnlyView::new(current_status.clone()), read_only_lock::std::ReadOnlyLock::new(execution_status.clone()), + shard_epoch, ) .await; @@ -6434,6 +6555,7 @@ impl Worker { initial_worker_metadata.clone(), read_only_lock::arc_swap::ReadOnlyView::new(initial_status.clone()), read_only_lock::std::ReadOnlyLock::new(execution_status.clone()), + shard_epoch, ) .await } else { @@ -6445,6 +6567,7 @@ impl Worker { initial_worker_metadata.clone(), read_only_lock::arc_swap::ReadOnlyView::new(initial_status.clone()), read_only_lock::std::ReadOnlyLock::new(execution_status.clone()), + shard_epoch, ) .await }; @@ -6825,6 +6948,58 @@ struct PendingWorkerInterrupt { unload_request: UnloadRequest, } +/// The shard epoch this executor currently holds for the agent's shard, if it holds one. +/// +/// `None` only when there is no assignment at all yet (before registration), or when the agent's +/// shard is not in it - in which case admission has already refused the work, and an oplog opened +/// without an epoch simply asserts nothing. +fn owned_shard_epoch(this: &T, agent_id: &AgentId) -> Option { + let assignment = this.shard_service().try_get_current_assignment()?; + let shard_id = ShardId::from_agent_id(agent_id, assignment.number_of_shards); + assignment.epoch_of(&shard_id) +} + +/// Why this executor is giving an agent up: it no longer owns the agent's shard. +/// +/// Distinct from [`UnloadReason`], which says why an agent left memory. An agent can be unloaded +/// for memory pressure and be back a moment later; a relinquished one is gone from this executor +/// and belongs to the shard's new owner. +#[derive(Clone, Debug)] +pub(crate) enum RelinquishReason { + /// A write to the agent's oplog was refused by the storage. Carries the fence when the write + /// path had it to hand; `None` when the loop only saw the classified interrupt. + Fenced(Option>), + /// The shard manager revoked the shard. + ShardRevoked, + /// A delivered assignment no longer contains the agent's shard. + ShardNotAssigned, +} + +impl RelinquishReason { + /// What anyone waiting on the agent is told. Every variant is one the worker service answers + /// by refreshing its routing table and retrying, so the invocation lands on the new owner + /// instead of failing. + pub(crate) fn to_error(&self) -> WorkerExecutorError { + match self { + RelinquishReason::Fenced(Some(fence)) => WorkerExecutorError::oplog_fenced( + fence.agent_id.clone(), + fence.expected_epoch.0, + fence.actual_epoch.map(|epoch| epoch.0), + ), + // The loop saw the classified interrupt without the fence details, or the shard was + // taken back explicitly. Either way the caller's move is the same. + RelinquishReason::Fenced(None) + | RelinquishReason::ShardRevoked + | RelinquishReason::ShardNotAssigned => WorkerExecutorError::ShardingNotReady, + } + } + + /// Which `OwnerFailureWinner` entity bodies are torn down with. + pub(crate) fn owner_failure(&self) -> OwnerFailureWinner { + OwnerFailureWinner::Lifecycle(InterruptKind::ShardLost) + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum UnloadReason { Deleting, @@ -6839,6 +7014,7 @@ pub(crate) enum UnloadReason { OutOfMemory, Panic, Restart, + ShardLost, Suspend, } @@ -6848,6 +7024,7 @@ impl UnloadReason { InterruptKind::Restart | InterruptKind::Jump => Self::Restart, InterruptKind::Suspend(_) => Self::Suspend, InterruptKind::Interrupt(_) => Self::Interrupt, + InterruptKind::ShardLost => Self::ShardLost, } } } @@ -6893,7 +7070,7 @@ impl PendingWorkerInterrupt { } else { match self.kind { InterruptKind::Restart | InterruptKind::Jump => RetryDecision::Immediate, - InterruptKind::Interrupt(_) => RetryDecision::None, + InterruptKind::Interrupt(_) | InterruptKind::ShardLost => RetryDecision::None, InterruptKind::Suspend(timestamp) => RetryDecision::TryStop(timestamp), } } @@ -8922,6 +9099,16 @@ mod tests { decision(InterruptKind::Suspend(suspend_timestamp), false), RetryDecision::TryStop(suspend_timestamp) ); + // A lost shard is terminal here: a retry in place would reopen the oplog with the same + // stale epoch, and the agent belongs to the shard's new owner now. + assert_eq!( + decision(InterruptKind::ShardLost, false), + RetryDecision::None + ); + assert_eq!( + UnloadReason::from_interrupt(InterruptKind::ShardLost), + UnloadReason::ShardLost + ); // Permit reacquisition overrides the kind-based decision for every kind. for kind in [ @@ -8929,6 +9116,7 @@ mod tests { InterruptKind::Jump, InterruptKind::Interrupt(Timestamp::now_utc()), InterruptKind::Suspend(Timestamp::now_utc()), + InterruptKind::ShardLost, ] { assert_eq!( decision(kind, true), @@ -8938,6 +9126,60 @@ mod tests { } } + #[test] + fn a_relinquished_agent_reports_an_error_its_caller_can_retry() { + let agent_id = AgentId { + component_id: golem_common::model::component::ComponentId::new(), + agent_id: "relinquished".to_string(), + }; + let fence = OplogFence { + agent_id, + expected_epoch: golem_common::model::ShardEpoch(3), + actual_epoch: Some(golem_common::model::ShardEpoch(4)), + }; + + // A fence the write path saw in full names both epochs, so an operator reading the log + // can tell which generation lost. + assert!(matches!( + RelinquishReason::Fenced(Some(Box::new(fence))).to_error(), + WorkerExecutorError::OplogFenced { + expected_epoch: 3, + actual_epoch: Some(4), + .. + } + )); + + // Every other shape gives the same answer a lapsed lease does: the worker service + // refreshes its routing table and retries on the owner. None of them may look like a + // plain invocation failure, or the caller would give up instead of moving. + for reason in [ + RelinquishReason::Fenced(None), + RelinquishReason::ShardRevoked, + RelinquishReason::ShardNotAssigned, + ] { + assert!( + matches!(reason.to_error(), WorkerExecutorError::ShardingNotReady), + "{reason:?} must be retriable on the new owner" + ); + } + } + + #[test] + fn giving_an_agent_up_never_looks_like_an_api_interrupt() { + // Entity bodies are torn down as `ShardLost`, not `Interrupt`: the agent was not + // interrupted through the Golem API, its shard moved. + for reason in [ + RelinquishReason::Fenced(None), + RelinquishReason::ShardRevoked, + RelinquishReason::ShardNotAssigned, + ] { + assert!(matches!( + reason.owner_failure(), + OwnerFailureWinner::Lifecycle(InterruptKind::ShardLost) + )); + } + } + #[test] fn interrupt_terminality_matrix() { fn terminal(kind: InterruptKind) -> bool { @@ -8953,6 +9195,7 @@ mod tests { assert!(!terminal(InterruptKind::Jump)); assert!(terminal(InterruptKind::Interrupt(Timestamp::now_utc()))); assert!(terminal(InterruptKind::Suspend(Timestamp::now_utc()))); + assert!(terminal(InterruptKind::ShardLost)); } #[test] diff --git a/golem-worker-executor/src/worker/state_actor.rs b/golem-worker-executor/src/worker/state_actor.rs index 52506dc132..eae7661679 100644 --- a/golem-worker-executor/src/worker/state_actor.rs +++ b/golem-worker-executor/src/worker/state_actor.rs @@ -52,11 +52,12 @@ use super::status::{calculate_last_known_status_with_checkpoint, update_status_with_new_entries}; use super::status_flusher::{AgentStatusFlusher, FlushReason}; use super::{ - PendingMemoryGrowth, UnloadReason, Worker, WorkerCommand, WorkerInstance, WorkerStatusMetric, + PendingMemoryGrowth, RelinquishReason, UnloadReason, Worker, WorkerCommand, WorkerInstance, + WorkerStatusMetric, }; use crate::services::linear_memory::LinearMemoryTracker; -use crate::services::oplog::{CommitLevel, Oplog}; -use crate::services::{All, HasConfig, HasSchedulerService}; +use crate::services::oplog::{CommitLevel, Oplog, OplogError, OplogFence}; +use crate::services::{All, HasActiveAgents, HasConfig, HasSchedulerService}; use crate::workerctx::WorkerCtx; use arc_swap::ArcSwap; use chrono::Utc; @@ -124,7 +125,7 @@ enum StatusJob { expected_result_generation: u64, expected_revert_generation: u64, instance_guard: OwnedMutexGuard, - done: oneshot::Sender, + done: oneshot::Sender>, }, /// Returns the published status after reattaching it when a jump or revert detached it. /// Serialization on the status queue prevents observing an in-flight status transition. @@ -254,11 +255,20 @@ impl WorkerStateActor { } => { complete_status_job( async { - state.oplog.add(*entry).await; - state - .commit_and_update_state(CommitLevel::Always, None) - .await; - state.ensure_status_attached().await; + match state.oplog.add(*entry).await { + Ok(_) => { + state + .commit_and_update_state(CommitLevel::Always, None) + .await; + state.ensure_status_attached().await; + } + // The shard has a new owner: give the agent up and leave no + // further trace in an oplog that is no longer ours. + Err(OplogError::Fenced(fence)) => { + state.relinquish_fenced_agent(fence) + } + Err(error) => panic!("oplog write: {error}"), + } }, done, ) @@ -282,17 +292,24 @@ impl WorkerStateActor { expected_result_generation, expected_revert_generation, ) { - return false; + return Ok(false); } drop(status); - state.oplog.add(*entry).await; + // Returned rather than reported as a moved version: the caller + // retries on `false`, and a fenced oplog refuses every retry. + if let Err(error) = state.oplog.add(*entry).await { + if let OplogError::Fenced(fence) = &error { + state.relinquish_fenced_agent(fence.clone()); + } + return Err(error); + } state .commit_and_update_state(CommitLevel::Always, None) .await; if let WorkerInstance::Running(running) = &*instance_guard { running.sender.send(WorkerCommand::WorkAvailable).unwrap(); } - true + Ok(true) }, done, ) @@ -429,7 +446,7 @@ impl WorkerStateActor { expected_result_generation: u64, expected_revert_generation: u64, instance_guard: OwnedMutexGuard, - ) -> bool { + ) -> Result { self.commit .run_status_job(|done| StatusJob::AppendInvocationIfVersion { entry: Box::new(entry), @@ -576,6 +593,27 @@ async fn complete_status_job(transaction: impl Future, done: ones } impl StatusState { + /// Gives the agent up after a background oplog write was refused because its shard moved. + /// + /// Spawned rather than awaited: this runs on the status task, which must never take the + /// worker's instance lock (callers holding that lock await status jobs), and the stop inside + /// [`Worker::relinquish`] does take it. Handing the stop to an independent task keeps that + /// discipline while still dropping the agent from this executor - which a bare + /// `mark_relinquished` would not do, because on a background path nothing else is unwinding + /// to carry the stop out. + fn relinquish_fenced_agent(&self, fence: OplogFence) { + let active_agents = self.deps.active_agents(); + let agent_id = self.owned_agent_id.agent_id.clone(); + tokio::spawn(async move { + active_agents + .relinquish_matching( + RelinquishReason::Fenced(Some(Box::new(fence))), + |candidate| candidate == &agent_id, + ) + .await; + }); + } + /// The commit + status-fold transaction. Commits the oplog, then either folds the newly /// committed entries into the published status or marks the status detached when it can no /// longer be incrementally computed (e.g. after a revert or a snapshot update). Returns @@ -585,7 +623,17 @@ impl StatusState { commit_level: CommitLevel, committed: Option>, ) -> bool { - let new_entries = self.oplog.commit(commit_level).await; + let new_entries = match self.oplog.commit(commit_level).await { + Ok(entries) => entries, + Err(OplogError::Fenced(fence)) => { + // Nothing was committed and nothing more can be. The `committed` sender is + // dropped rather than signalled: a fenced commit is not a commit, and every + // awaiter already reads a dropped sender as "no commit observed". + self.relinquish_fenced_agent(fence); + return false; + } + Err(error) => panic!("oplog write: {error}"), + }; if let Some(committed) = committed { let _ = committed.send(()); } diff --git a/golem-worker-executor/src/worker/status.rs b/golem-worker-executor/src/worker/status.rs index 6597522336..f6ccfb8d26 100644 --- a/golem-worker-executor/src/worker/status.rs +++ b/golem-worker-executor/src/worker/status.rs @@ -1733,6 +1733,7 @@ mod test { use golem_common::base_model::OplogIndex; use golem_common::base_model::environment_plugin_grant::EnvironmentPluginGrantId; use golem_common::base_model::oplog::{CardInstallFailure, QueuedCardEvent}; + use golem_common::model::ShardEpoch; use golem_common::model::account::AccountId; use golem_common::model::agent::{AgentMode, Principal}; use golem_common::model::application::ApplicationId; @@ -2936,6 +2937,7 @@ mod test { trace_states: vec![], invocation_context: vec![], wallet_pin: None, + shard_epoch: None, }, move |mut status| { status.current_idempotency_key = Some(idempotency_key); @@ -3368,6 +3370,7 @@ mod test { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -3380,6 +3383,7 @@ mod test { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } @@ -3392,6 +3396,7 @@ mod test { _initial_worker_metadata: AgentMetadata, _last_known_status: read_only_lock::arc_swap::ReadOnlyView, _execution_status: read_only_lock::std::ReadOnlyLock, + _shard_epoch: Option, ) -> Arc { unreachable!() } diff --git a/golem-worker-executor/tests/indexed_storage.rs b/golem-worker-executor/tests/indexed_storage.rs index a6ac70342a..6a0c5a3cb9 100644 --- a/golem-worker-executor/tests/indexed_storage.rs +++ b/golem-worker-executor/tests/indexed_storage.rs @@ -16,6 +16,7 @@ use async_trait::async_trait; use bytes::Bytes; use golem_common::config::{DbPostgresConfig, RedisConfig}; use golem_common::model::AgentId; +use golem_common::model::ShardEpoch; use golem_common::model::agent::AgentMode; use golem_common::model::component::ComponentId; use golem_common::redis::RedisPool; @@ -43,6 +44,10 @@ use uuid::Uuid; #[async_trait] trait GetIndexedStorage: Debug { async fn get_indexed_storage(&self) -> Arc; + + /// Whether this backend is expected to enforce the shard-epoch fence. Stated here rather than + /// read off the storage so that a backend silently losing its fence fails a test. + fn expects_fencing(&self) -> bool; } struct InMemoryIndexedStorageWrapper; @@ -55,6 +60,10 @@ impl Debug for InMemoryIndexedStorageWrapper { #[async_trait] impl GetIndexedStorage for InMemoryIndexedStorageWrapper { + fn expects_fencing(&self) -> bool { + false + } + async fn get_indexed_storage(&self) -> Arc { let kvs = InMemoryIndexedStorage::new(); Arc::new(kvs) @@ -80,6 +89,10 @@ impl Debug for RedisIndexedStorageWrapper { #[async_trait] impl GetIndexedStorage for RedisIndexedStorageWrapper { + fn expects_fencing(&self) -> bool { + false + } + async fn get_indexed_storage(&self) -> Arc { let random_prefix = Uuid::new_v4(); let redis_pool = RedisPool::configured(&RedisConfig { @@ -132,6 +145,10 @@ impl Debug for SqliteIndexedStorageWrapper { #[async_trait] impl GetIndexedStorage for SqliteIndexedStorageWrapper { + fn expects_fencing(&self) -> bool { + true + } + async fn get_indexed_storage(&self) -> Arc { let tempdir = tempfile::tempdir().unwrap(); let database = tempdir @@ -177,6 +194,10 @@ impl Debug for MultiSqliteIndexedStorageWrapper { #[async_trait] impl GetIndexedStorage for MultiSqliteIndexedStorageWrapper { + fn expects_fencing(&self) -> bool { + true + } + async fn get_indexed_storage(&self) -> Arc { let tempdir = tempfile::tempdir().unwrap(); let path = tempdir.path().to_path_buf(); @@ -206,6 +227,10 @@ impl Debug for PostgresIndexedStorageWrapper { #[async_trait] impl GetIndexedStorage for PostgresIndexedStorageWrapper { + fn expects_fencing(&self) -> bool { + true + } + async fn get_indexed_storage(&self) -> Arc { let db_name = format!("idx_{}", Uuid::new_v4().simple()); @@ -327,7 +352,15 @@ async fn postgres_singleton_append_many_preserves_storage_contract( let value = Bytes::from_static(&[0, 255, 17, 3]); for ns in [primary, compressed] { storage - .append_many("svc", "api", "entity", &ns.ns, "singleton", Arc::from([])) + .append_many( + "svc", + "api", + "entity", + &ns.ns, + "singleton", + Arc::from([]), + None, + ) .await .unwrap(); assert!( @@ -344,6 +377,7 @@ async fn postgres_singleton_append_many_preserves_storage_contract( &ns.ns, "singleton", Arc::from([(17, value.clone())]), + None, ) .await .unwrap(); @@ -388,7 +422,8 @@ async fn postgres_singleton_append_many_preserves_storage_contract( "entity", &ns.ns, "singleton", - Arc::from([(u64::MAX, value.clone())]) + Arc::from([(u64::MAX, value.clone())]), + None, ) .await, Err(IndexedStorageError::Other(_)) @@ -409,7 +444,8 @@ async fn postgres_singleton_append_many_preserves_storage_contract( "entity", &primary.ns, "singleton", - Arc::from([(17, Bytes::from_static(b"replacement"))]) + Arc::from([(17, Bytes::from_static(b"replacement"))]), + None, ) .await, Err(IndexedStorageError::Conflict(_)) @@ -446,6 +482,7 @@ async fn postgres_append_many_rolls_back_across_statement_chunks( "atomic", 1025, b"original".to_vec(), + None, ) .await .unwrap(); @@ -455,7 +492,7 @@ async fn postgres_append_many_rolls_back_across_statement_chunks( .into(); assert!(matches!( storage - .append_many("svc", "api", "entity", &ns.ns, "atomic", pairs) + .append_many("svc", "api", "entity", &ns.ns, "atomic", pairs, None) .await, Err(IndexedStorageError::Conflict(_)) )); @@ -482,7 +519,7 @@ async fn exists_append( let value1 = "value1".as_bytes().to_vec(); let result1 = is.exists("svc", "api", ns.ns.clone(), key1).await.unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1, None) .await .unwrap(); let result2 = is.exists("svc", "api", ns.ns.clone(), key1).await.unwrap(); @@ -504,9 +541,18 @@ async fn namespaces_are_separate( let key1 = "key1"; let value1 = "value1".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns1.ns.clone(), key1, 1, value1) - .await - .unwrap(); + is.append( + "svc", + "api", + "entity", + ns1.ns.clone(), + key1, + 1, + value1, + None, + ) + .await + .unwrap(); let result = is.exists("svc", "api", ns2.ns.clone(), key1).await.unwrap(); assert_eq!(result, false); @@ -535,6 +581,7 @@ async fn can_append_and_get( key1, 1, value1.clone(), + None, ) .await .unwrap(); @@ -546,6 +593,7 @@ async fn can_append_and_get( key1, 2, value2.clone(), + None, ) .await .unwrap(); @@ -557,6 +605,7 @@ async fn can_append_and_get( key1, 3, value3.clone(), + None, ) .await .unwrap(); @@ -583,11 +632,11 @@ async fn append_cannot_overwrite( let value1 = "value1".as_bytes().to_vec(); let value2 = "value2".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1, None) .await .unwrap(); let result1 = is - .append("svc", "api", "entity", ns.ns.clone(), key1, 1, value2) + .append("svc", "api", "entity", ns.ns.clone(), key1, 1, value2, None) .await; assert!(result1.is_err()); @@ -615,6 +664,7 @@ async fn append_can_skip( key1, 4, value1.clone(), + None, ) .await .unwrap(); @@ -626,6 +676,7 @@ async fn append_can_skip( key1, 8, value2.clone(), + None, ) .await .unwrap(); @@ -653,11 +704,11 @@ async fn length( let value2 = "value2".as_bytes().to_vec(); let result1 = is.length("svc", "api", ns.ns.clone(), key1).await.unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 4, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 4, value1, None) .await .unwrap(); let result2 = is.length("svc", "api", ns.ns.clone(), key1).await.unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 8, value2) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 8, value2, None) .await .unwrap(); let result3 = is.length("svc", "api", ns.ns.clone(), key1).await.unwrap(); @@ -709,10 +760,10 @@ async fn scan_with_no_pattern_single_paged( let value1 = "value1".as_bytes().to_vec(); let value2 = "value2".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1, None) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key2, 1, value2) + is.append("svc", "api", "entity", ns.ns.clone(), key2, 1, value2, None) .await .unwrap(); @@ -760,6 +811,7 @@ async fn scan_with_no_pattern_paginated( key1, 1, value1.clone(), + None, ) .await .unwrap(); @@ -771,6 +823,7 @@ async fn scan_with_no_pattern_paginated( key1, 2, value2.clone(), + None, ) .await .unwrap(); @@ -782,6 +835,7 @@ async fn scan_with_no_pattern_paginated( key2, 1, value2.clone(), + None, ) .await .unwrap(); @@ -793,6 +847,7 @@ async fn scan_with_no_pattern_paginated( key3, 3, value3.clone(), + None, ) .await .unwrap(); @@ -871,13 +926,13 @@ async fn scan_with_prefix_pattern_single_paged( let value2 = "value2".as_bytes().to_vec(); let value3 = "value3".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1, None) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key2, 1, value2) + is.append("svc", "api", "entity", ns.ns.clone(), key2, 1, value2, None) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key3, 1, value3) + is.append("svc", "api", "entity", ns.ns.clone(), key3, 1, value3, None) .await .unwrap(); @@ -917,13 +972,13 @@ async fn scan_with_prefix_pattern_paginated( let value2 = "value2".as_bytes().to_vec(); let value3 = "value3".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1, None) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key2, 1, value2) + is.append("svc", "api", "entity", ns.ns.clone(), key2, 1, value2, None) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key3, 1, value3) + is.append("svc", "api", "entity", ns.ns.clone(), key3, 1, value3, None) .await .unwrap(); @@ -982,7 +1037,7 @@ async fn exists_append_delete( let value1 = "value1".as_bytes().to_vec(); let result1 = is.exists("svc", "api", ns.ns.clone(), key1).await.unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 1, value1, None) .await .unwrap(); is.delete("svc", "api", ns.ns.clone(), key1).await.unwrap(); @@ -1005,9 +1060,18 @@ async fn delete_is_per_namespace( let key1 = "key1"; let value1 = "value1".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns1.ns.clone(), key1, 1, value1) - .await - .unwrap(); + is.append( + "svc", + "api", + "entity", + ns1.ns.clone(), + key1, + 1, + value1, + None, + ) + .await + .unwrap(); is.delete("svc", "api", ns2.ns.clone(), key1).await.unwrap(); let result = is.exists("svc", "api", ns1.ns.clone(), key1).await.unwrap(); @@ -1057,6 +1121,7 @@ async fn first( key1, 5, value1.clone(), + None, ) .await .unwrap(); @@ -1068,6 +1133,7 @@ async fn first( key1, 7, value2.clone(), + None, ) .await .unwrap(); @@ -1106,6 +1172,7 @@ async fn last( key1, 5, value1.clone(), + None, ) .await .unwrap(); @@ -1117,6 +1184,7 @@ async fn last( key1, 7, value2.clone(), + None, ) .await .unwrap(); @@ -1155,6 +1223,7 @@ async fn closest_low( key1, 5, value1.clone(), + None, ) .await .unwrap(); @@ -1166,6 +1235,7 @@ async fn closest_low( key1, 7, value2.clone(), + None, ) .await .unwrap(); @@ -1204,6 +1274,7 @@ async fn closest_match( key1, 5, value1.clone(), + None, ) .await .unwrap(); @@ -1215,6 +1286,7 @@ async fn closest_match( key1, 7, value2.clone(), + None, ) .await .unwrap(); @@ -1253,6 +1325,7 @@ async fn closest_mid( key1, 5, value1.clone(), + None, ) .await .unwrap(); @@ -1264,6 +1337,7 @@ async fn closest_mid( key1, 7, value2.clone(), + None, ) .await .unwrap(); @@ -1294,10 +1368,10 @@ async fn closest_high( .closest("svc", "api", "entity", ns.ns.clone(), key1, 10) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 5, value1) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 5, value1, None) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 7, value2) + is.append("svc", "api", "entity", ns.ns.clone(), key1, 7, value2, None) .await .unwrap(); let result2 = is @@ -1332,6 +1406,7 @@ async fn drop_prefix_no_match( key1, 10, value1.clone(), + None, ) .await .unwrap(); @@ -1343,6 +1418,7 @@ async fn drop_prefix_no_match( key1, 11, value2.clone(), + None, ) .await .unwrap(); @@ -1354,6 +1430,7 @@ async fn drop_prefix_no_match( key1, 12, value3.clone(), + None, ) .await .unwrap(); @@ -1392,6 +1469,7 @@ async fn drop_prefix_partial( key1, 10, value1.clone(), + None, ) .await .unwrap(); @@ -1403,6 +1481,7 @@ async fn drop_prefix_partial( key1, 11, value2.clone(), + None, ) .await .unwrap(); @@ -1414,6 +1493,7 @@ async fn drop_prefix_partial( key1, 12, value3.clone(), + None, ) .await .unwrap(); @@ -1444,23 +1524,391 @@ async fn drop_prefix_full( let value2 = "value2".as_bytes().to_vec(); let value3 = "value3".as_bytes().to_vec(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 10, value1) + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key1, + 10, + value1, + None, + ) + .await + .unwrap(); + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key1, + 11, + value2, + None, + ) + .await + .unwrap(); + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key1, + 12, + value3, + None, + ) + .await + .unwrap(); + + is.drop_prefix("svc", "api", ns.ns.clone(), key1, 20) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 11, value2) + let result = is + .read("svc", "api", "entity", ns.ns.clone(), key1, 1, 100) .await .unwrap(); - is.append("svc", "api", "entity", ns.ns.clone(), key1, 12, value3) + + assert_eq!(result, vec![]); +} + +// --------------------------------------------------------------------------------------------- +// The shard-epoch fence. +// +// Every test below runs against all five backends. The ones that cannot fence (redis, in-memory) +// must behave exactly as they did before the epoch argument existed - accept the write and ignore +// the epoch - so each test asserts both halves rather than being skipped for them. +// --------------------------------------------------------------------------------------------- + +fn assert_fenced( + result: Result<(), IndexedStorageError>, + expected_epoch: u64, + actual_epoch: Option, +) { + match result { + Err(IndexedStorageError::Fenced { + expected, actual, .. + }) => { + assert_eq!(expected, ShardEpoch(expected_epoch), "expected epoch"); + assert_eq!(actual, actual_epoch.map(ShardEpoch), "stored epoch"); + } + other => panic!("expected a Fenced error, got {other:?}"), + } +} + +#[test] +#[tracing::instrument] +async fn append_with_the_recorded_epoch_is_accepted( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let is = is.get_indexed_storage().await; + let key = "fence-match"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(7)) .await .unwrap(); + is.append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([ + (1, Bytes::from_static(b"a")), + (2, Bytes::from_static(b"b")), + (3, Bytes::from_static(b"c")), + ]), + Some(ShardEpoch(7)), + ) + .await + .unwrap(); - is.drop_prefix("svc", "api", ns.ns.clone(), key1, 20) + assert_eq!( + is.length("svc", "api", ns.ns.clone(), key).await.unwrap(), + 3 + ); +} + +#[test] +#[tracing::instrument] +async fn a_stale_epoch_append_is_refused_and_writes_nothing( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let fencing = is.expects_fencing(); + let is = is.get_indexed_storage().await; + let key = "fence-stale"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(8)) .await .unwrap(); let result = is - .read("svc", "api", "entity", ns.ns.clone(), key1, 1, 100) + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([ + (1, Bytes::from_static(b"a")), + (2, Bytes::from_static(b"b")), + (3, Bytes::from_static(b"c")), + ]), + Some(ShardEpoch(7)), + ) + .await; + + let length = is.length("svc", "api", ns.ns.clone(), key).await.unwrap(); + if fencing { + assert_fenced(result, 7, Some(8)); + // The whole batch is rolled back, not the tail of it. + assert_eq!(length, 0, "a refused batch must leave no entry behind"); + } else { + result.unwrap(); + assert_eq!(length, 3); + } +} + +#[test] +#[tracing::instrument] +async fn an_append_without_a_recorded_epoch_is_refused( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let fencing = is.expects_fencing(); + let is = is.get_indexed_storage().await; + let key = "fence-absent"; + + // No upsert. Epoch 0 is a perfectly valid epoch, so this also pins that an absent row is not + // silently treated as zero. + let result = is + .append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 1, + b"a".to_vec(), + Some(ShardEpoch(0)), + ) + .await; + + let length = is.length("svc", "api", ns.ns.clone(), key).await.unwrap(); + if fencing { + assert_fenced(result, 0, None); + assert_eq!(length, 0); + } else { + result.unwrap(); + assert_eq!(length, 1); + } +} + +#[test] +#[tracing::instrument] +async fn an_unfenced_append_ignores_the_recorded_epoch( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let is = is.get_indexed_storage().await; + let key = "fence-none"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(9)) .await .unwrap(); + // `None` asserts nothing: it is what the archive layers and generic callers pass. + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 1, + b"a".to_vec(), + None, + ) + .await + .unwrap(); - assert_eq!(result, vec![]); + assert_eq!( + is.length("svc", "api", ns.ns.clone(), key).await.unwrap(), + 1 + ); +} + +#[test] +#[tracing::instrument] +async fn the_recorded_epoch_only_ever_climbs( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let fencing = is.expects_fencing(); + let is = is.get_indexed_storage().await; + let key = "fence-monotonic"; + + // Rising and repeated epochs are accepted ... + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(5)) + .await + .unwrap(); + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(5)) + .await + .unwrap(); + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(9)) + .await + .unwrap(); + + // ... a falling one is not, or a zombie could re-open at its stale epoch and un-fence itself + // against the current owner. + let result = is + .upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(8)) + .await; + + if fencing { + assert_fenced(result, 8, Some(9)); + // and the rejected upsert left the record alone + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 1, + b"a".to_vec(), + Some(ShardEpoch(9)), + ) + .await + .unwrap(); + assert_fenced( + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 2, + b"b".to_vec(), + Some(ShardEpoch(8)), + ) + .await, + 8, + Some(9), + ); + } else { + result.unwrap(); + } +} + +#[test] +#[tracing::instrument] +async fn deleting_the_recorded_epoch_fences_later_writes( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let fencing = is.expects_fencing(); + let is = is.get_indexed_storage().await; + let key = "fence-deleted"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(3)) + .await + .unwrap(); + is.delete_oplog_metadata("svc", "api", ns.ns.clone(), key) + .await + .unwrap(); + // Idempotent: deleting again is not an error. + is.delete_oplog_metadata("svc", "api", ns.ns.clone(), key) + .await + .unwrap(); + + let result = is + .append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 1, + b"a".to_vec(), + Some(ShardEpoch(3)), + ) + .await; + + if fencing { + assert_fenced(result, 3, None); + } else { + result.unwrap(); + } +} + +#[test] +#[tracing::instrument] +async fn the_backend_reports_whether_it_fences( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, +) { + let expected = is.expects_fencing(); + let is = is.get_indexed_storage().await; + assert_eq!(is.supports_epoch_fencing(), expected); +} + +#[test] +#[tracing::instrument] +async fn a_failed_batch_leaves_no_partial_write( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + // The transactional backends are exactly the fencing ones, and this is what makes "the fence + // is checked once per batch" true rather than "once per entry": a backend that loops single + // appends would leave the entries before the failure behind. + if !is.expects_fencing() { + return; + } + let is = is.get_indexed_storage().await; + let key = "batch-atomicity"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(1)) + .await + .unwrap(); + is.append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(1, Bytes::from_static(b"a")), (2, Bytes::from_static(b"b"))]), + Some(ShardEpoch(1)), + ) + .await + .unwrap(); + + // id 1 already exists, so the second entry of this batch violates the primary key. + let result = is + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([ + (3, Bytes::from_static(b"c")), + (1, Bytes::from_static(b"dup")), + (4, Bytes::from_static(b"d")), + ]), + Some(ShardEpoch(1)), + ) + .await; + + assert!(result.is_err(), "a duplicate id must fail the batch"); + assert_eq!( + is.length("svc", "api", ns.ns.clone(), key).await.unwrap(), + 2, + "the failed batch must not have written its first entry" + ); } diff --git a/golem-worker-service/src/service/worker/client.rs b/golem-worker-service/src/service/worker/client.rs index ae3b741563..61833056b0 100644 --- a/golem-worker-service/src/service/worker/client.rs +++ b/golem-worker-service/src/service/worker/client.rs @@ -1768,6 +1768,14 @@ impl WorkerClient for WorkerExecutorWorkerClient { }, |outcome| match outcome { OneShotInvocationSessionResult::Success(output) => Ok(output), + // A routing miss, retried on the shard's owner like the typed failure an + // executor sends for the same condition after accepting. + OneShotInvocationSessionResult::Rejected(rejected) + if rejected.reason + == InvocationRejectionReason::ShardingNotReady as i32 => + { + Err(WorkerExecutorError::ShardingNotReady.into()) + } OneShotInvocationSessionResult::Rejected(rejected) => { Err(decode_invocation_rejection(rejected).into()) } @@ -2453,7 +2461,9 @@ mod one_shot_session_tests { #[cfg(test)] mod rejection_mapping_tests { - use super::{WorkerClient, WorkerExecutorWorkerClient, decode_invocation_rejection}; + use super::{ + WorkerClient, WorkerExecutorWorkerClient, WorkerServiceError, decode_invocation_rejection, + }; use futures::{Stream, stream}; use golem_api_grpc::proto::golem::schema::{SchemaValue, schema_value}; use golem_api_grpc::proto::golem::shardmanager::{ @@ -2486,6 +2496,7 @@ mod rejection_mapping_tests { use std::net::Ipv4Addr; use std::pin::Pin; use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use test_r::test; use tokio::net::TcpListener; use tokio_stream::wrappers::TcpListenerStream; @@ -2600,8 +2611,13 @@ mod rejection_mapping_tests { } } - #[derive(Clone)] - struct RejectingExecutor; + /// Rejects every invocation as `NotFound`, after first rejecting `routing_misses` of them as + /// having reached an executor that does not own the agent's shard. + #[derive(Clone, Default)] + struct RejectingExecutor { + routing_misses: Arc, + calls: Arc, + } macro_rules! unimplemented_unary { ($name:ident, $request:ty, $response:ty) => { @@ -2749,12 +2765,27 @@ mod rejection_mapping_tests { )) => (start.idempotency_key, start.agent_id), other => panic!("expected invocation start, got {other:?}"), }; + self.calls.fetch_add(1, Ordering::SeqCst); + let routing_miss = self + .routing_misses + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| { + left.checked_sub(1) + }) + .is_ok(); + let (reason, error) = if routing_miss { + ( + InvocationRejectionReason::ShardingNotReady, + "0 is not in shards []", + ) + } else { + (InvocationRejectionReason::NotFound, "agent not found") + }; Ok(Response::new(Box::pin(stream::iter([Ok( InvocationResponse { response: Some(invocation_response::Response::Rejected( InvocationRejected { - reason: InvocationRejectionReason::NotFound as i32, - error: "agent not found".to_string(), + reason: reason as i32, + error: error.to_string(), idempotency_key, agent_id, component_revision: None, @@ -2765,14 +2796,15 @@ mod rejection_mapping_tests { } } - #[test] - async fn unary_not_found_rejection_preserves_the_public_error_category() { + /// Invokes an agent through a worker service whose only executor is `executor`, and returns + /// the error the invocation ends with. + async fn invoke_against(executor: RejectingExecutor) -> WorkerServiceError { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); tokio::spawn(async move { tonic::transport::Server::builder() .add_service( - WorkerExecutorServer::new(RejectingExecutor) + WorkerExecutorServer::new(executor) .accept_compressed(CompressionEncoding::Gzip) .send_compressed(CompressionEncoding::Gzip), ) @@ -2816,7 +2848,7 @@ mod rejection_mapping_tests { agent_id: "missing".to_string(), }; - let error = client + client .invoke_agent( &agent_id, Some("run".to_string()), @@ -2836,7 +2868,12 @@ mod rejection_mapping_tests { None, ) .await - .unwrap_err(); + .unwrap_err() + } + + #[test] + async fn unary_not_found_rejection_preserves_the_public_error_category() { + let error = invoke_against(RejectingExecutor::default()).await; let public_error: AgentError = error.into(); assert!( @@ -2844,6 +2881,31 @@ mod rejection_mapping_tests { "InvocationRejected(NotFound) must remain a public not-found error, got {public_error:?}" ); } + + #[test] + async fn a_routing_miss_rejection_is_retried_rather_than_surfaced() { + // An executor that has just lost the agent's shard rejects before accepting. The worker + // service has to retry that on the shard's owner - here the same fake, answering the second + // time - rather than fail the invocation with the first rejection. + let executor = RejectingExecutor { + routing_misses: Arc::new(AtomicUsize::new(1)), + ..Default::default() + }; + let calls = executor.calls.clone(); + + let error = invoke_against(executor).await; + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "the routing miss must be retried, exactly once" + ); + let public_error: AgentError = error.into(); + assert!( + matches!(public_error.error, Some(agent_error::Error::NotFound(_))), + "the retried call's answer must be the one surfaced, got {public_error:?}" + ); + } } #[cfg(test)] diff --git a/integration-tests/tests/sharding.rs b/integration-tests/tests/sharding.rs index 619c913e65..d0f09824d8 100644 --- a/integration-tests/tests/sharding.rs +++ b/integration-tests/tests/sharding.rs @@ -29,7 +29,8 @@ mod tests { use golem_common::model::plugin_registration::{ OplogProcessorPluginSpec, PluginRegistrationCreation, PluginSpecDto, }; - use golem_common::model::{AgentStatus, IdempotencyKey, OplogIndex}; + use golem_common::model::{AgentId, AgentStatus, IdempotencyKey, OplogIndex}; + use golem_common::schema::SchemaValue; use golem_common::tracing::{TracingConfig, init_tracing_with_default_debug_env_filter}; use golem_common::{agent_id, data_value}; use golem_test_framework::components::rdb::DbInfo; @@ -71,6 +72,9 @@ mod tests { pub async fn create_deps() -> EnvBasedTestDependencies { let deps = EnvBasedTestDependencies::new(EnvBasedTestDependenciesConfig { number_of_shards_override: Some(16), + // The shortest lease the shard manager accepts, so that a paused executor is seen to + // lose its shards well inside a test's timeout. + shard_lease_duration_override: Some(Duration::from_secs(30)), ..EnvBasedTestDependenciesConfig::new() }) .await @@ -277,6 +281,156 @@ mod tests { chaos.await.unwrap(); } + #[test] + #[timeout(300000)] + // Not `#[flaky]`, unlike the scenarios above: what this pins is that a duplicate never + // happens, and a retry would hide one that only happens some of the time. + async fn an_executor_paused_past_its_lease_cannot_finish_the_invocations_it_started( + deps: &EnvBasedTestDependencies, + cluster_control: &WorkerExecutorClusterControlStub, + _tracing: &Tracing, + ) { + // Under the executor's `suspend_after` (10s by default), so the sleep runs in flight instead + // of suspending the agent: the executors freeze mid-invocation, which is the case a lease + // alone cannot stop - a frozen executor wakes up still holding work it can try to finish. + const DELAY_MILLIS: u64 = 8_000; + + deps.reset(cluster_control, 16).await; + let admin = deps.admin().await; + let (_, env) = admin.app_and_env().await.unwrap(); + let component = admin + .component(&env.id, "it_agent_counters_release") + .name("it:agent-counters") + .store() + .await + .unwrap(); + + let mut agents = Vec::new(); + for i in 1..=8 { + let parsed_agent_id = agent_id!("InstantiationGrowthCounter", format!("fenced-{i}")); + let agent_id = admin + .start_agent(&component.id, parsed_agent_id.clone()) + .await + .unwrap(); + agents.push((parsed_agent_id, agent_id)); + } + + let mut invocations = JoinSet::new(); + for (parsed_agent_id, _) in &agents { + let user = deps.admin().await; + let component = component.clone(); + let parsed_agent_id = parsed_agent_id.clone(); + invocations.spawn( + async move { + let result = user + .invoke_and_await_agent_with_key( + &component, + &parsed_agent_id, + &IdempotencyKey::fresh(), + "delayed_increment", + data_value!(DELAY_MILLIS), + ) + .await; + (parsed_agent_id, result) + } + .in_current_span(), + ); + } + + // Every invocation is inside its sleep now, on whichever executor owns its agent. Freezing + // all executors but one leaves the survivor as the only one to take their shards over, so + // nothing here needs to know which executor owned which agent. + tokio::time::sleep(Duration::from_secs(2)).await; + let started = cluster_control.started_indices().await; + let (survivor, frozen) = started + .split_first() + .expect("the reset starts every executor"); + info!("Pausing worker executors {frozen:?}, keeping {survivor}"); + for idx in frozen { + cluster_control.pause(*idx).await; + } + + // Past the frozen executors' leases, so that their shards are granted to the survivor at a + // higher epoch and it recovers and finishes their invocations itself, and past the delay, + // so that the frozen executors' own sleeps are over the moment they wake. + tokio::time::sleep(Duration::from_secs(45)).await; + + // Thawed, the frozen executors carry on from exactly where they stopped, still holding + // invocations the survivor now owns. For each one, either the executor gives the agent up + // when it re-registers, or it finishes the sleep first and the fence refuses its write. + info!("Resuming worker executors {frozen:?}"); + for idx in frozen { + cluster_control.resume(*idx).await; + } + + while let Some(joined) = + tokio::time::timeout(Duration::from_secs(180), invocations.join_next()) + .await + .expect("Timed out waiting for the invocations to finish") + { + let (parsed_agent_id, result) = joined.unwrap(); + let value = result + .unwrap_or_else(|err| panic!("{parsed_agent_id}: invocation failed: {err:?}")) + .into_return_value() + .unwrap_or_else(|| panic!("{parsed_agent_id}: expected a return value")); + assert_eq!( + value, + SchemaValue::U32(1), + "{parsed_agent_id}: the delayed increment must be applied exactly once" + ); + } + + // No executor may have died on the way: a write the fence let through would land in an + // oplog the survivor is writing too, and a conflicting append there is fatal to the + // executor that makes it - which can be the rightful owner. Checked through the health + // endpoint rather than process liveness: an aborting process stays "running" until the OS + // has finished writing its crash report, which can outlast this test. + for idx in cluster_control.started_indices().await { + assert!( + cluster_control.is_serving(idx).await, + "worker executor {idx} stopped serving during the test" + ); + } + + for (parsed_agent_id, agent_id) in &agents { + assert_eq!( + count_completions_of(&admin, agent_id, "delayed_increment").await, + 1, + "{parsed_agent_id}: exactly one executor may record the invocation's completion" + ); + // The agent's state agrees: one increment from the delayed call, one from this. + let next = admin + .invoke_and_await_agent(&component, parsed_agent_id, "increment", data_value!()) + .await + .unwrap() + .into_return_value() + .unwrap_or_else(|| panic!("{parsed_agent_id}: expected a return value")); + assert_eq!( + next, + SchemaValue::U32(2), + "{parsed_agent_id}: the delayed increment must not have been applied twice" + ); + } + } + + /// Counts the completions `agent_id`'s oplog records for `method`. Scoped to one method on + /// purpose: an agent's own initialization completes asynchronously after `start_agent` returns, + /// so any count taken over every method races it. + async fn count_completions_of(user: &impl TestDsl, agent_id: &AgentId, method: &str) -> usize { + user.get_oplog(agent_id, OplogIndex::INITIAL) + .await + .unwrap() + .into_iter() + .filter(|entry| { + matches!( + &entry.entry, + PublicOplogEntry::AgentInvocationFinished(params) + if params.method_name.as_deref() == Some(method) + ) + }) + .count() + } + async fn coordinated_scenario( deps: &EnvBasedTestDependencies, cluster_control: &WorkerExecutorClusterControlStub, diff --git a/local-run/start.sh b/local-run/start.sh index 2387d597a1..e6538a930b 100644 --- a/local-run/start.sh +++ b/local-run/start.sh @@ -17,7 +17,7 @@ fi LOCAL_RUN_DIR="${GOLEM_DIR}/local-run" rm -rf "${LOCAL_RUN_DIR}/data/shard-manager" -mkdir -pv "${LOCAL_RUN_DIR}/data/redis" "${LOCAL_RUN_DIR}/data/shard-manager" "${LOCAL_RUN_DIR}/logs" +mkdir -pv "${LOCAL_RUN_DIR}/data/redis" "${LOCAL_RUN_DIR}/data/shard-manager" "${LOCAL_RUN_DIR}/data/worker-executor" "${LOCAL_RUN_DIR}/logs" # start redis # Redis persistence isn't needed for local-run, and misconfigured snapshotting can force Redis into @@ -138,6 +138,10 @@ GOLEM__SHARD_MANAGER__HOST="localhost" \ GOLEM__SHARD_MANAGER__PORT=${SHARD_MANAGER_GRPC_PORT} \ GOLEM__SHARD_MANAGER__RETRIES__MAX_ATTEMPTS=10 \ GOLEM__SHARD_MANAGER__RETRIES__MIN_DELAY=1s \ +GOLEM__INDEXED_STORAGE__TYPE="Sqlite" \ +GOLEM__INDEXED_STORAGE__CONFIG__DATABASE="../local-run/data/worker-executor/golem_indexed.sqlite" \ +GOLEM__INDEXED_STORAGE__CONFIG__MAX_CONNECTIONS=10 \ +GOLEM__INDEXED_STORAGE__CONFIG__FOREIGN_KEYS=false \ ../target/debug/worker-executor & worker_executor_pid=$! From f5e3ed9087dd3f134541be8bdd104c90dc1958fe Mon Sep 17 00:00:00 2001 From: Aditya Salunkhe Date: Wed, 16 Sep 2026 11:16:59 +0530 Subject: [PATCH 2/6] Fixes --- docs/src/content/next/deploy.mdx | 10 +- .../v1/shard_manager_service.proto | 20 +- golem-common/src/base_model/oplog/mod.rs | 10 +- golem-common/src/model/oplog/tests.rs | 71 +- ..._invocation_started_before_shard_epoch.bin | Bin 0 -> 182 bytes .../src/oplog/debug_oplog_constructor.rs | 7 +- .../src/clients/shard_manager.rs | 37 +- golem-service-base/src/db/mod.rs | 2 +- golem-shard-manager/src/grpc.rs | 34 +- golem-shard-manager/src/sharding/model.rs | 244 +++- .../src/sharding/shard_management.rs | 189 ++- golem-shard-manager/tests/shard_management.rs | 558 ++++++++- .../src/components/shard_manager/mod.rs | 8 - .../src/components/shard_manager/spawned.rs | 7 - .../src/components/worker_executor/spawned.rs | 110 +- golem-test-framework/src/config/benchmark.rs | 1 - golem-test-framework/src/config/env.rs | 5 - golem-worker-executor-test-utils/src/lib.rs | 74 +- .../src/durable_host/concurrent/call.rs | 31 +- .../src/durable_host/durable_stream.rs | 283 ++++- .../src/durable_host/golem/v1x.rs | 17 +- golem-worker-executor/src/durable_host/mod.rs | 131 ++- .../src/durable_host/p3/http/replay.rs | 12 +- .../src/durable_host/p3/http/send.rs | 101 ++ .../src/durable_host/p3/http/test_support.rs | 16 +- .../src/durable_host/rdbms/mod.rs | 14 +- golem-worker-executor/src/grpc/mod.rs | 37 +- golem-worker-executor/src/lib.rs | 44 +- golem-worker-executor/src/model/mod.rs | 130 ++- .../src/services/active_agents/mod.rs | 67 +- .../src/services/oplog/mod.rs | 46 +- .../src/services/oplog/multilayer.rs | 39 +- .../src/services/oplog/plugin.rs | 146 ++- .../src/services/oplog/primary.rs | 315 +++-- .../src/services/oplog/tests.rs | 1018 +++++++++++++++++ golem-worker-executor/src/services/quota.rs | 2 + golem-worker-executor/src/services/rpc.rs | 42 +- golem-worker-executor/src/services/shard.rs | 172 ++- .../src/services/shard_manager.rs | 436 ++++++- .../src/services/worker_fork.rs | 91 +- .../src/storage/indexed/mod.rs | 9 +- .../src/storage/indexed/postgres.rs | 35 +- .../src/storage/indexed/sqlite.rs | 5 +- .../src/worker/invocation_loop.rs | 214 +++- golem-worker-executor/src/worker/mod.rs | 359 +++++- .../src/worker/state_actor.rs | 73 +- golem-worker-executor/tests/active_agents.rs | 92 ++ golem-worker-executor/tests/api.rs | 293 ++++- .../tests/indexed_storage.rs | 301 +++-- .../src/service/worker/client.rs | 412 +++++-- integration-tests/tests/sharding.rs | 120 +- local-run/start.sh | 4 +- 52 files changed, 5790 insertions(+), 704 deletions(-) create mode 100644 golem-common/tests/fixtures/oplog/agent_invocation_started_before_shard_epoch.bin diff --git a/docs/src/content/next/deploy.mdx b/docs/src/content/next/deploy.mdx index 8eaf886fe8..6c6e2b6af0 100644 --- a/docs/src/content/next/deploy.mdx +++ b/docs/src/content/next/deploy.mdx @@ -55,9 +55,15 @@ See also: [configuration](https://github.com/golemcloud/golem/blob/main/golem-wo - **Local mode** (`persistence.type = "Postgres"` or `"Sqlite"`) stores the shard state and the quota state in one SQL database. There is no leader election, so **exactly one instance may run**. - **Distributed mode** (`persistence.type = "Etcd"`) stores the shard state in etcd. Any number of replicas may run: one wins an etcd lease campaign and drives all topology decisions while the rest stand by. Quota enforcement is not available in this mode yet: the quota state has no store outside the SQL database, so every quota lease request fails. -Every worker executor holds a **lease** on its shard assignment and renews it with the shard manager. A lease lasts `shard_lease_duration` (1m by default) and an executor renews at a third of the time it has left, so roughly every 20s. The shard manager refuses to start with a lease shorter than 30s and warns about one shorter than 1m: the deadline it allows each renewal call never drops below 10s, so a shorter lease cannot fit the three renewal attempts a lease is meant to get. Each renewal carries the epoch of every shard the executor holds; a shard's epoch is its ownership generation, advanced only when the shard moves to another executor and never by a renewal. A renewal whose claim does not match the shard manager's view is renewed all the same, and the response carries the shard manager's set, which the executor adopts exactly as it would an assignment push: pushes deliver a change at once, and renewals guarantee it arrives within a third of the lease even if the push was lost. The shard manager records a shard's new owner before it tells anybody about it, so a rebalance whose write is refused changes nothing at all, and every delivery carries the revision of the state it was read from. An executor ignores a delivery older than one it has already applied, so a push and a renewal that cross on the network cannot leave the older set in place. An executor that stops renewing loses its lease, and the shard manager reclaims and redistributes those shards on its next pass, which runs at least every third of `shard_lease_duration` even in a cluster where nothing else is happening; a crashed or unreachable executor's shards are therefore re-homed within about one lease duration plus one pass. The lease reaches the executor as the time remaining on it rather than as an absolute time, so the two machines' clocks are never compared and skew between them can neither lengthen nor shorten it. The executor anchors that time to the moment it sent the request the lease answers, so time the reply spends in flight comes off its copy of the lease and is never added to it, and its copy lapses no later than the shard manager's. An assignment push carries the shard set and its revision only: it never extends a lease. The lease is the one place the two machines' clocks must agree on a rate rather than on a value: an executor whose clock runs slow relative to the shard manager's keeps admitting for that fraction of a lease past the shard manager reclaiming it, which is milliseconds under ordinary `ntp` or `chrony` discipline and seconds while such a daemon is slewing off a large offset, so an executor should not be started until its clock is synchronised. An executor whose own copy of the lease has expired stops **admitting** new work for its shards and reports that sharding is not ready, which a worker service retries after refreshing its routing table - invocations already running are not interrupted. A graceful stop - including the `SIGTERM` an orchestrator sends to stop a pod - deregisters the executor instead of waiting for its lease to expire, so its shards move on the next pass. A shard manager restart or failover re-grants the lease of every executor that still answers the startup health check, so a restart never evicts a healthy cluster: only executors that fail that check are removed. An executor whose own copy of the lease lapsed during the outage resumes admitting work on its next successful renewal rather than on the push that follows the re-grant, and neither a re-grant nor a renewal ever moves an executor's deadline earlier, so a `shard_lease_duration` reduced across a restart takes effect only once the new length outruns the deadline each executor was last told. +Every worker executor holds a **lease** on its shard assignment and renews it with the shard manager. A lease lasts `shard_lease_duration` (1m by default) and an executor renews at a third of the time it has left, so roughly every 20s. The shard manager refuses to start with a lease shorter than 30s and warns about one shorter than 1m: the deadline it allows each renewal call never drops below 10s, so a shorter lease cannot fit the three renewal attempts a lease is meant to get. Each renewal carries the epoch of every shard the executor holds; a shard's epoch is its ownership generation, advanced when the shard moves to another executor and otherwise only to repair a shard manager state that lost history, as described under oplog fencing below. A renewal whose claim does not match the shard manager's view is renewed all the same, and the response carries the shard manager's set, which the executor adopts exactly as it would an assignment push: pushes deliver a change at once, and renewals guarantee it arrives within a third of the lease even if the push was lost. The shard manager records a shard's new owner before it tells anybody about it, so a rebalance whose write is refused changes nothing at all, and every delivery carries the revision of the state it was read from. An executor ignores a delivery older than one it has already applied, so a push and a renewal that cross on the network cannot leave the older set in place. An executor that stops renewing loses its lease, and the shard manager reclaims and redistributes those shards on its next pass, which runs at least every third of `shard_lease_duration` even in a cluster where nothing else is happening; a crashed or unreachable executor's shards are therefore re-homed within about one lease duration plus one pass. The lease reaches the executor as the time remaining on it rather than as an absolute time, so the two machines' clocks are never compared and skew between them can neither lengthen nor shorten it. The executor anchors that time to the moment it sent the request the lease answers, so time the reply spends in flight comes off its copy of the lease and is never added to it, and its copy lapses no later than the shard manager's. An assignment push carries the shard set and its revision only: it never extends a lease. The lease is the one place the two machines' clocks must agree on a rate rather than on a value: an executor whose clock runs slow relative to the shard manager's keeps admitting for that fraction of a lease past the shard manager reclaiming it, which is milliseconds under ordinary `ntp` or `chrony` discipline and seconds while such a daemon is slewing off a large offset, so an executor should not be started until its clock is synchronised. An executor whose own copy of the lease has expired stops **admitting** new work for its shards and reports that sharding is not ready, which a worker service retries after refreshing its routing table - invocations already running are not interrupted. A graceful stop - including the `SIGTERM` an orchestrator sends to stop a pod - deregisters the executor instead of waiting for its lease to expire, so its shards move on the next pass. A shard manager restart or failover re-grants the lease of every executor that still answers the startup health check, so a restart never evicts a healthy cluster: only executors that fail that check are removed. An executor whose own copy of the lease lapsed during the outage resumes admitting work on its next successful renewal rather than on the push that follows the re-grant, and neither a re-grant nor a renewal ever moves an executor's deadline earlier, so a `shard_lease_duration` reduced across a restart takes effect only once the new length outruns the deadline each executor was last told. -The lease bounds how long a lost shard keeps being served, but it cannot stop an executor that has already lost one from finishing a write it had started. That is what **oplog fencing** is for. Each agent's oplog records the shard epoch allowed to write it, and every batch of entries is checked against that record inside the same database transaction that inserts them, so a refused write leaves nothing behind. An executor whose epoch is behind is turned away at the storage rather than discovering the problem later: that one agent stops there, and the worker service resumes it on the shard's new owner. The record only ever moves forward - an executor re-granted the shard at a higher epoch takes over, and one holding a stale epoch cannot claim it back - and an absent record fences too, because the record is written before an oplog's first entry and removed before its last. Fencing is enforced by the PostgreSQL and SQLite indexed storage backends. Redis and the in-memory backend cannot enforce it, so an executor configured with one of those **and** a shard manager refuses to start rather than run unprotected; set `indexed_storage.type` to `Postgres`, `Sqlite`, `KVStoreSqlite`, `MultiSqlite` or `KVStoreMultiSqlite`. Single-shard deployments and the debugging service are exempt, because nothing can take a shard away from them. During a rolling upgrade the fence is complete only once every executor runs the new code and each agent has been opened once under it. Ephemeral agents are not fenced: their oplogs are never replayed, so a duplicate there is a duplicated observability record rather than duplicated state. +The lease bounds how long a lost shard keeps being served, but it cannot stop an executor that has already lost one from finishing a write it had started. That is what **oplog fencing** is for. Each agent's oplog records the shard epoch allowed to write it, and every batch of entries is checked against that record inside the same database transaction that inserts them, so a refused write leaves nothing behind. An executor whose epoch is behind is turned away at the storage rather than discovering the problem later: that one agent stops there, and the worker service resumes it on the shard's new owner. The record only ever moves forward - an executor re-granted the shard at a higher epoch takes over, and one holding a stale epoch cannot claim it back - and an absent record fences an open oplog too, because the record is written before an oplog's first entry and removed before its last. That protection covers an oplog whose record exists. Deleting an agent removes its record and forgets the epoch with it: an executor that still has the oplog open is refused, but an oplog without a record - a new agent, a deleted agent created again, or one last written before this release - is claimed by whichever epoch opens it first, so for those only the lease stops an executor that has lost the shard. Ephemeral agents are not fenced: their oplogs are never replayed, so a duplicate there is a duplicated observability record rather than duplicated state. + +Fencing is enforced by the PostgreSQL and SQLite indexed storage backends. Redis and the in-memory backend cannot enforce it, so an executor configured with one of those **and** a shard manager refuses to start rather than run unprotected; set `indexed_storage.type` to `Postgres`, `Sqlite`, `KVStoreSqlite`, `MultiSqlite` or `KVStoreMultiSqlite`. `KVStoreSqlite` and `KVStoreMultiSqlite` derive their location from the key-value storage, so they also require `key_value_storage.type` to be `Sqlite` or `MultiSqlite` respectively; any other combination fails at startup. The check applies to every executor that registers with a shard manager, whatever its shard count; only an executor that runs without one, such as the debugging service, is exempt. + +A shard epoch only means something against the shard manager state that minted it. A state that is wiped, replaced or restored from a backup mints epochs below the ones the oplogs already record, and every write at those would be refused, so the shard manager repairs its record from what the executors tell it, without an operator. An executor that kept running sends the epochs it holds on its next renewal, or - when the state no longer lists the executor and refuses that renewal - on the re-registration that follows. The shard manager raises its record to them, and a shard it has meanwhile given to another executor stays with that owner at a new epoch one past the reported one, so two executors never write a shard's oplogs at the same epoch. When every executor restarted as well, none of them holds the old epochs, and the oplogs supply them instead: an executor refused a write reports the epoch the oplog recorded on its next renewal, and the shard manager gives whichever executor owns that shard a new epoch one past it. Either way the affected agents resume at the new epoch within about one renewal interval, while the worker service retries their requests. An oplog nobody opens keeps its old record until it is opened, and that first refused open repairs it the same way. To clear the old records up front instead, or if an agent is still refused after several renewals, stop every executor, run `DELETE FROM oplog_metadata` on the indexed storage - for `MultiSqlite` and `KVStoreMultiSqlite`, on every `*-oplog-*.db` file under the root directory - and start the executors again; each oplog records its owner's epoch when it is next opened. Run the delete only while no executor is running, because an oplog without a record accepts whichever executor opens it first. + +Upgrade the shard manager and the worker services before the worker executors. An upgraded executor reports a request that reached it without its shard as a routing miss, which only an upgraded worker service retries, and it sends the epochs the repair above relies on in fields an earlier shard manager ignores. Once any executor runs this release, only roll forward: it adds the table that records these epochs to the indexed storage, and an executor from an earlier release refuses to start against indexed storage that has applied that migration, so an older executor restarted mid-rollout, or a rollback of the executor image, fails at startup. During a rolling upgrade the fence is complete only once every executor runs the new code and each agent has been opened once under it. Distributed mode talks to etcd over **plaintext HTTP** and without authentication. Every entry in `persistence.config.endpoints` must start with `http://`; a replica configured with anything else refuses to start. etcd therefore has to be reachable unauthenticated, either on a trusted network or through a TLS-terminating proxy running alongside the shard manager, with the endpoints pointing at that proxy. diff --git a/golem-api-grpc/proto/golem/shardmanager/v1/shard_manager_service.proto b/golem-api-grpc/proto/golem/shardmanager/v1/shard_manager_service.proto index 486bc4cc4d..0b8b29f374 100644 --- a/golem-api-grpc/proto/golem/shardmanager/v1/shard_manager_service.proto +++ b/golem-api-grpc/proto/golem/shardmanager/v1/shard_manager_service.proto @@ -50,6 +50,15 @@ message RegisterRequest { // the same address refreshes the existing lease rather than creating a second // one. An empty or non-UUID value is rejected before any state is touched. string executor_id = 3; + // The shards, with their epochs, this executor held under an earlier + // executor_id that the manager answered ShardLeaseNotFound for; empty on a + // process's first registration. Evidence only: it never assigns a shard. An + // epoch ahead of the manager's record means the manager's state lost history + // - a wiped or replaced store no longer lists the earlier executor_id, so the + // renewal that would have repaired it was refused - and the manager raises + // its record so that the epochs it mints next clear the oplog rows written at + // the old ones. A malformed entry is rejected before any state is touched. + repeated golem.shardmanager.ShardEpochEntry previous_shard_epochs = 4; } message RegisterResponse { @@ -97,10 +106,19 @@ message ShardLease { // manager's view is renewed all the same, and the response carries the set the // manager holds for this executor, which the executor adopts. That makes the // renewal a guaranteed second delivery path for a push that was lost, and it -// means a renewal never advances an epoch. +// means a renewal never advances an epoch against a record that kept its +// history. message RenewShardLeaseRequest { string executor_id = 1; repeated golem.shardmanager.ShardEpochEntry shard_epochs = 2; + // Epochs recorded on oplogs this executor was refused writes to, keyed by + // the shard each agent routes to. Evidence only: never an assignment. Above + // the manager's record they mean its state lost history, and every owner of + // the shard, this executor included, is minted one past them; at or below + // the record they move nothing. Applied before this request's shard_epochs. + // Reported on every renewal until one is granted. A malformed entry is + // rejected before any state is touched. + repeated golem.shardmanager.ShardEpochEntry fenced_shard_epochs = 3; } message RenewShardLeaseResponse { diff --git a/golem-common/src/base_model/oplog/mod.rs b/golem-common/src/base_model/oplog/mod.rs index e1a73d3c31..159742f13c 100644 --- a/golem-common/src/base_model/oplog/mod.rs +++ b/golem-common/src/base_model/oplog/mod.rs @@ -201,10 +201,12 @@ oplog_entry! { invocation_context: Vec, wallet_pin: Option, /// The shard epoch this executor held for the agent's shard when the invocation - /// started. Raw only - it is a record of which ownership generation produced the - /// entry, for operators and oplog-processor plugins reading a divergence, not - /// something the agent's own history should expose. `None` for entries written - /// before the fence existed, and for oplogs opened without an epoch to assert. + /// started. Raw only - a record of which ownership generation produced the entry, + /// visible to executor-internal code and to anyone reading oplog storage directly. + /// The public oplog and the WIT oplog records handed to oplog-processor plugins both + /// drop it, and converting a WIT raw entry back yields `None`. `None` also for + /// entries written before the fence existed, and for oplogs opened without an epoch + /// to assert. shard_epoch: Option, } public { diff --git a/golem-common/src/model/oplog/tests.rs b/golem-common/src/model/oplog/tests.rs index 352fede345..cfea4b1355 100644 --- a/golem-common/src/model/oplog/tests.rs +++ b/golem-common/src/model/oplog/tests.rs @@ -1286,7 +1286,8 @@ fn shard_epoch_protobuf_roundtrip_and_legacy_default() { } } - // And on the raw protobuf, which is the oplog-processor-plugin channel. + // And on the raw protobuf, the form oplog-processor batches travel in between services. The + // plugin guest itself receives the WIT form, which drops the epoch. let entry = started(Some(9)); let mut raw_proto: golem_api_grpc::proto::golem::worker::RawOplogEntry = entry.clone().try_into().unwrap(); @@ -1317,6 +1318,74 @@ fn shard_epoch_protobuf_roundtrip_and_legacy_default() { } } +#[test] +fn agent_invocation_started_written_before_shard_epoch_still_decodes() { + // Encoded by a6cb8e36a, where this variant's evolution list still ended at `wallet_pin`. + // These are the bytes replay reads from an oplog written before the fence. The round trips + // above encode and decode with the same evolution list, so they cannot catch a misordered + // list; a populated `wallet_pin` and trailing fields decoding intact here can. + let bytes = include_bytes!( + "../../../tests/fixtures/oplog/agent_invocation_started_before_shard_epoch.bin" + ); + let decoded: OplogEntry = crate::serialization::deserialize(bytes).unwrap(); + + match decoded { + OplogEntry::AgentInvocationStarted { + timestamp, + idempotency_key, + payload, + trace_id, + trace_states, + invocation_context, + wallet_pin, + shard_epoch, + } => { + assert_eq!(timestamp, Timestamp::from(1_767_323_045_000u64)); + assert_eq!( + idempotency_key, + IdempotencyKey::new("shard-epoch-legacy".to_string()) + ); + match payload { + OplogPayload::SerializedInline { bytes, .. } => { + assert_eq!( + crate::serialization::deserialize::(&bytes) + .unwrap(), + AgentInvocationPayload::AgentMethod { + method_name: "test".to_string(), + input: SchemaValue::Record { fields: Vec::new() }, + principal: Principal::anonymous(), + scope_card: None, + } + ); + } + other => panic!("expected an inline payload, got {other:?}"), + } + assert_eq!( + trace_id, + TraceId( + std::num::NonZeroU128::new(0x0123_4567_89ab_cdef_fedc_ba98_7654_3210).unwrap() + ) + ); + assert_eq!(trace_states, vec!["vendor=fixture".to_string()]); + assert!(invocation_context.is_empty()); + assert_eq!( + wallet_pin, + Some(InvocationWalletPin { + wallet_token: WalletVersionToken { + wallet_id_hash: [0x42; 32], + generation: 73, + }, + pinned_card_ids: vec![CardId(Uuid::from_u128(1)), CardId(Uuid::from_u128(2))], + scope_card_id: Some(CardId(Uuid::from_u128(3))), + }) + ); + // Absent, not epoch zero - zero is a real epoch. + assert_eq!(shard_epoch, None); + } + other => panic!("expected raw invocation-started entry, got {other:?}"), + } +} + #[test] fn invocation_wallet_pin_protobuf_roundtrip_and_legacy_defaults() { let pinned_card_ids = vec![CardId::new(), CardId::new()]; diff --git a/golem-common/tests/fixtures/oplog/agent_invocation_started_before_shard_epoch.bin b/golem-common/tests/fixtures/oplog/agent_invocation_started_before_shard_epoch.bin new file mode 100644 index 0000000000000000000000000000000000000000..a7411849cf2e1ca9c0722316c27aa06b0ef8e181 GIT binary patch literal 182 zcmZQ(U}0RqxQCH}fpK=t%AO9D;*7+i6y4N<{NxPXoYeHhj`f?_7QM{l7cAW|V~(2{6f&rRJsN7ulv|R+N+$r7{2wRB$2yfaZXJCkupz NlR&*FfC*iI8359=BisN0 literal 0 HcmV?d00001 diff --git a/golem-debugging-service/src/oplog/debug_oplog_constructor.rs b/golem-debugging-service/src/oplog/debug_oplog_constructor.rs index 512c2aaa4c..a1d6851068 100644 --- a/golem-debugging-service/src/oplog/debug_oplog_constructor.rs +++ b/golem-debugging-service/src/oplog/debug_oplog_constructor.rs @@ -17,7 +17,7 @@ use crate::oplog::debug_oplog::DebugOplog; use async_trait::async_trait; use golem_common::model::agent::AgentMode; use golem_common::model::oplog::{OplogEntry, OplogIndex}; -use golem_common::model::{AgentMetadata, AgentStatusRecord, OwnedAgentId}; +use golem_common::model::{AgentMetadata, AgentStatusRecord, OwnedAgentId, ShardEpoch}; use golem_common::read_only_lock; use golem_worker_executor::model::ExecutionStatus; use golem_worker_executor::services::oplog::{Oplog, OplogConstructor, OplogService}; @@ -64,6 +64,11 @@ impl CreateDebugOplogConstructor { #[async_trait] impl OplogConstructor for CreateDebugOplogConstructor { + fn shard_epoch(&self) -> Option { + // A debugging session discards every write, so it asserts no epoch. + None + } + async fn create_oplog(self, _close: Box) -> Arc { let inner = if let Some(initial_entry) = self.initial_entry { self.inner diff --git a/golem-service-base/src/clients/shard_manager.rs b/golem-service-base/src/clients/shard_manager.rs index 9b545460ea..8c110055cf 100644 --- a/golem-service-base/src/clients/shard_manager.rs +++ b/golem-service-base/src/clients/shard_manager.rs @@ -56,11 +56,19 @@ pub trait ShardManager: Send + Sync { /// UUID it generated at startup. Idempotent: the same `executor_id` at the /// same address refreshes the existing shard lease rather than creating a /// second one. + /// + /// `previous_shard_epochs` is the set this process held under an earlier + /// `executor_id` the manager answered `LeaseNotFound` for, and is empty on + /// a first registration. It is evidence, not a request: it never assigns a + /// shard, and only raises the manager's recorded epochs where its state has + /// lost history, so the epochs it mints next clear the oplog rows this + /// process wrote before. async fn register( &self, port: u16, pod_name: Option, executor_id: Uuid, + previous_shard_epochs: BTreeMap, ) -> Result; /// Extends this executor's shard lease. `shard_epochs` is the set the @@ -68,10 +76,17 @@ pub trait ShardManager: Send + Sync { /// claim that does not match the manager's view is renewed all the same, /// and the returned lease carries the manager's set, which the caller /// adopts exactly as it would an `AssignShards` push. + /// + /// `fenced_shard_epochs` are the epochs recorded on oplogs this executor + /// was refused writes to, keyed by shard. Evidence, like + /// `previous_shard_epochs` on `register`: above the manager's record they + /// mean its state lost history, and every owner of the shard is minted one + /// past them; at or below it they move nothing. async fn renew_shard_lease( &self, executor_id: Uuid, shard_epochs: BTreeMap, + fenced_shard_epochs: BTreeMap, ) -> Result; /// Releases the shard lease on a graceful shutdown. Lenient by contract: a @@ -285,14 +300,21 @@ impl ShardManager for GrpcShardManager { port: u16, pod_name: Option, executor_id: Uuid, + previous_shard_epochs: BTreeMap, ) -> Result { with_retries( "shard_manager", "register", Some(format!("{pod_name:?}")), &self.retries, - &(self.client.clone(), port, pod_name, executor_id), - |(client, port, pod_name, executor_id)| { + &( + self.client.clone(), + port, + pod_name, + executor_id, + previous_shard_epochs, + ), + |(client, port, pod_name, executor_id, previous_shard_epochs)| { Box::pin(async move { let (sent_at, response) = client .call("register", move |client| { @@ -300,6 +322,11 @@ impl ShardManager for GrpcShardManager { port: *port as i32, pod_name: pod_name.clone(), executor_id: executor_id.to_string(), + previous_shard_epochs: shard_epochs_to_proto( + previous_shard_epochs + .iter() + .map(|(shard_id, epoch)| (*shard_id, *epoch)), + ), }; Box::pin(async move { let (sent_at, response) = issued(client.register(request)).await; @@ -339,6 +366,7 @@ impl ShardManager for GrpcShardManager { &self, executor_id: Uuid, shard_epochs: BTreeMap, + fenced_shard_epochs: BTreeMap, ) -> Result { let (sent_at, response) = self .client @@ -350,6 +378,11 @@ impl ShardManager for GrpcShardManager { .iter() .map(|(shard_id, epoch)| (*shard_id, *epoch)), ), + fenced_shard_epochs: shard_epochs_to_proto( + fenced_shard_epochs + .iter() + .map(|(shard_id, epoch)| (*shard_id, *epoch)), + ), }; Box::pin(async move { let (sent_at, response) = issued(client.renew_shard_lease(request)).await; diff --git a/golem-service-base/src/db/mod.rs b/golem-service-base/src/db/mod.rs index ea9e5cca16..6ecb2671d6 100644 --- a/golem-service-base/src/db/mod.rs +++ b/golem-service-base/src/db/mod.rs @@ -83,7 +83,7 @@ pub trait Pool: Debug + Sync + Clone { Err(err) => { warn!( svc_name, api_name, error = ?err, - "Rolling back, transaction failed with repo error", + "Rolling back transaction, closure returned an error", ); // If rollback fails, we still return the original error, but log the rollback error diff --git a/golem-shard-manager/src/grpc.rs b/golem-shard-manager/src/grpc.rs index 603174ea8d..4f20956ea2 100644 --- a/golem-shard-manager/src/grpc.rs +++ b/golem-shard-manager/src/grpc.rs @@ -56,11 +56,22 @@ impl ShardManagerServiceImpl { executor_id: ExecutorId, pod: Pod, pod_name: Option, + previous_claim: BTreeMap, ) -> Result { - debug!(executor_id = %executor_id, addr = %pod, "Received request to register executor"); + debug!( + executor_id = %executor_id, + addr = %pod, + previous_shards = previous_claim.len(), + "Received request to register executor" + ); let ack = self .shard_management - .register_executor(executor_id, ExecutorAddr::from(pod), pod_name) + .register_executor_with_previous_claim( + executor_id, + ExecutorAddr::from(pod), + pod_name, + previous_claim, + ) .await?; debug!(executor_id = %executor_id, addr = %pod, "Registered executor"); Ok(ack) @@ -100,11 +111,14 @@ impl ShardManagerService for ShardManagerServiceImpl { .ok_or_else(|| tonic::Status::invalid_argument("missing source IP"))? .ip(); - let request = request.into_inner(); + let mut request = request.into_inner(); - // Before anything touches the state: an executor that cannot name itself has no identity to - // renew or deregister a lease with. + // Both before anything touches the state: an executor that cannot name itself has no + // identity to renew or deregister a lease with, and a carried claim the manager cannot + // decode is not evidence it can weigh. let executor_id = parse_executor_id(&request.executor_id)?; + let previous_claim = + parse_shard_epochs(std::mem::take(&mut request.previous_shard_epochs))?; let record = recorded_grpc_api_request!( "register", @@ -117,7 +131,7 @@ impl ShardManagerService for ShardManagerServiceImpl { let pod = make_pod(source_ip, request.port)?; let response = self - .register_internal(executor_id, pod, request.pod_name) + .register_internal(executor_id, pod, request.pod_name, previous_claim) .instrument(record.span.clone()) .await; @@ -150,14 +164,16 @@ impl ShardManagerService for ShardManagerServiceImpl { ) -> Result, tonic::Status> { let request = request.into_inner(); - // Both before any state is touched: an executor that cannot name itself has no lease to - // renew, and a claim the manager cannot decode is not one it can validate. + // All before any state is touched: an executor that cannot name itself has no lease to + // renew, and a claim or a fenced epoch the manager cannot decode is not evidence it can + // weigh. let executor_id = parse_executor_id(&request.executor_id)?; let claimed = parse_shard_epochs(request.shard_epochs)?; + let fenced = parse_shard_epochs(request.fenced_shard_epochs)?; let result = match self .shard_management - .renew_shard_lease(executor_id, claimed) + .renew_shard_lease_with_fenced_epochs(executor_id, claimed, fenced) .await { Ok(grant) => golem::shardmanager::v1::renew_shard_lease_response::Result::Success( diff --git a/golem-shard-manager/src/sharding/model.rs b/golem-shard-manager/src/sharding/model.rs index f1e17fc0c9..191236ac65 100644 --- a/golem-shard-manager/src/sharding/model.rs +++ b/golem-shard-manager/src/sharding/model.rs @@ -487,20 +487,67 @@ impl ShardLeaseState { /// That case has to be repaired, because [`Self::shard_epochs`] is the only record of it. A /// fresh state starts the epochs at zero while the executors, and the oplog rows their writes /// are fenced against, still hold higher ones; since an epoch only climbs when a shard changes - /// owner, the fence would refuse those agents for good. The renewal carries the executor's - /// whole set, so it is the one moment the cluster can tell the manager what it forgot. + /// owner, the fence would refuse those agents for good. An executor's whole set reaches the + /// manager at two moments, and each repairs one kind of loss. A renewal repairs a state + /// restored from a backup that still lists the renewing executor. A state that was wiped or + /// replaced does not list it, so the renewal is refused as a lease not found, and the + /// re-registration that follows carries the set the executor held under its earlier id. A + /// process that restarted holds no set to carry, so a store lost while every executor restarted + /// as well is repaired by the first executor refused a write to one of those oplogs, on its + /// next renewal - see [`Self::raise_epoch_floor_past`] - and an oplog nobody opens keeps its + /// rows until it is opened. /// - /// Only `executor_id`'s own shards are repaired from its claim; a claim on a shard the - /// manager has given to somebody else is corrected by the grant, never adopted. + /// `executor_id`'s own shards, and unassigned ones, take the claimed epoch. A claim on a shard + /// the manager has given to somebody else is never adopted. At or below the record it is an + /// executor that missed a push, and the grant corrects it. Above the record it proves the + /// claimant held that epoch before the history was lost - the oplog rows it wrote are fenced + /// there, and they refuse the owner's writes at the lower epoch - so the owner is minted one + /// past the claim. The owner keeps the shard and the claimant is not given it. /// /// The assignment moves with the high-water and never apart from it - [`Self::check_invariants`] - /// requires the two to agree - and the corrected grant the renewal returns is what tells the - /// current owner its new epoch. The value only ever climbs, so this cannot walk an epoch back - /// to one a stale writer still holds. + /// requires the two to agree. The grant the caller returns tells the claimant its epochs; an + /// owner re-minted above another executor's claim hears only from a push, which the caller owes + /// it. The value only ever climbs, so this cannot walk an epoch back to one a + /// stale writer still holds. Returns every shard whose epoch moved, re-minted ones included. pub fn raise_epoch_floor( &mut self, executor_id: ExecutorId, claimed: &BTreeMap, + ) -> Vec { + self.raise_epoch_floor_for(Some(executor_id), claimed) + } + + /// Raises the recorded epoch of every assigned shard `stored` names to one past the stored + /// value, and reports the shards that moved. + /// + /// `stored` is what refused oplog writes found on the rows. Ahead of the record it proves the + /// state lost history, exactly as a claim ahead of it does - see [`Self::raise_epoch_floor`] - + /// but nothing proves the reporter wrote those rows, so every assigned entry, the reporter's + /// own included, is minted one past it: a generation nobody has held. An unassigned shard's + /// high-water rises to the stored value, so its next mint lands one past it. At or below the + /// record it is the ordinary loser of a shard move - the new owner recorded its epoch and the + /// old one was refused - and moves nothing. + /// + /// It does not commute with [`Self::raise_epoch_floor`] when a claim on the reporter's own + /// shard equals the report. Claim first records the claim, and the report then sits at the + /// record; report first mints one past it, and the claim is then below the record. Callers + /// that hold both apply the report first, so a report at or above the claim always ends one + /// past it. A claim that already reached the state in an earlier request is at the record when + /// an equal report arrives, and that tie is the equality this cannot tell from a shard move. + pub fn raise_epoch_floor_past( + &mut self, + stored: &BTreeMap, + ) -> Vec { + self.raise_epoch_floor_for(None, stored) + } + + /// The one rule behind [`Self::raise_epoch_floor`] and [`Self::raise_epoch_floor_past`]: an + /// entry `holder` owns takes the epoch itself, any other assigned entry is minted one past it, + /// and an unassigned shard's high-water takes it. `None` holds nothing. + fn raise_epoch_floor_for( + &mut self, + holder: Option, + claimed: &BTreeMap, ) -> Vec { let mut raised = Vec::new(); for (shard_id, claimed_epoch) in claimed { @@ -509,22 +556,9 @@ impl ShardLeaseState { if !self.contains_shard(*shard_id) { continue; } - // A claim on a shard the manager has given to somebody else is an executor that - // missed a push, not evidence about that shard's epoch. Stamping the claim onto the - // owner's entry would put two live executors on one `(shard, epoch)` - the pair the - // fence cannot tell apart, which is the whole point of the epoch. The grant corrects - // that claim instead, like any other stale entry. An unassigned shard has no entry to - // corrupt, and raising its high-water only makes the next mint start above the epoch - // the claim proves is already in use. - if self - .shard_assignments - .get(shard_id) - .is_some_and(|entry| entry.executor_id != executor_id) - { - continue; - } - // Against the high-water, so the floor only ever climbs: a value below one this shard - // has already reached is not evidence of anything. + // Against the high-water, so the floor only ever climbs: a value at or below one this + // shard has already reached is not evidence of anything. On a shard somebody else + // owns, that is the ordinary missed push, and the grant corrects it. if self .shard_epochs .get(shard_id) @@ -532,9 +566,19 @@ impl ShardLeaseState { { continue; } - self.shard_epochs.insert(*shard_id, *claimed_epoch); + // Stamping the claim itself onto another executor's entry would put two live + // executors on one `(shard, epoch)` - the pair the fence cannot tell apart, which is + // the whole point of the epoch. One past it is a generation nobody has held, and it + // clears the claimant's oplog rows. An unassigned shard has no entry to corrupt, and + // raising its high-water only makes the next mint start above the epoch the claim + // proves is already in use. + let epoch = match self.shard_assignments.get(shard_id) { + Some(entry) if Some(entry.executor_id) != holder => claimed_epoch.next(), + _ => *claimed_epoch, + }; + self.shard_epochs.insert(*shard_id, epoch); if let Some(entry) = self.shard_assignments.get_mut(shard_id) { - entry.epoch = *claimed_epoch; + entry.epoch = epoch; } raised.push(*shard_id); } @@ -1301,41 +1345,157 @@ mod tests { } #[test] - fn a_claim_on_another_executors_shard_never_raises_it() { - // Executor 1 holds shard 0; executor 2 missed the push that took it away and still claims - // it, at an epoch above the record. Adopting that would stamp executor 2's epoch onto - // executor 1's assignment, leaving both of them live on `(shard 0, epoch 9)` - which is - // exactly the pair the oplog fence cannot separate. + fn a_claim_ahead_on_another_executors_shard_re_mints_its_owner_above_it() { + // Executor 1 holds shard 0 at epoch 0 in a state that lost history; executor 2 held it at + // epoch 9 before the loss and still claims it. Adopting that would stamp executor 2's + // epoch onto executor 1's assignment, leaving both of them live on `(shard 0, epoch 9)` - + // exactly the pair the oplog fence cannot separate. Left at 0, executor 1 would have every + // write refused by the rows executor 2 wrote at 9. So the owner is minted one past it. let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[1])]); assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); - let poaching = BTreeMap::from([(shard(0), ShardEpoch(9))]); - assert!( - shard_state - .raise_epoch_floor(executor(2), &poaching) - .is_empty() + let ahead = BTreeMap::from([(shard(0), ShardEpoch(9))]); + assert_eq!( + shard_state.raise_epoch_floor(executor(2), &ahead), + vec![shard(0)] ); - assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); assert_eq!( shard_state .shard_assignments .get(&shard(0)) .map(|e| e.executor_id), Some(executor(1)), - "the owner was not changed either" + "the claimant was given the shard" ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(10))); assert_eq!( shard_state.shard_epochs.get(&shard(0)), - Some(&ShardEpoch(0)) + Some(&ShardEpoch(10)) ); + assert!(shard_state.check_invariants().is_ok()); - // The owner's own claim at the same epoch is still repaired. + // A claim at the record is an executor that missed a push, not evidence: nothing moves. + let at_record = BTreeMap::from([(shard(0), ShardEpoch(10))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &at_record) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(10))); + + // Nor does the owner's own claim at the epoch it was re-minted above. + assert!( + shard_state + .raise_epoch_floor(executor(1), &ahead) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(10))); + assert!(shard_state.check_invariants().is_ok()); + } + + #[test] + fn a_fenced_epoch_re_mints_every_owner_above_it_the_reporters_own_included() { + // Writes were refused by rows at epoch 3 on shards 0, 1 and 2, in a state that lost + // history. Nothing says who wrote those rows, so whoever reported them, executor 1's shard + // is minted past them like executor 2's: stamping 3 onto an entry could leave its owner on + // the rows' `(shard, epoch)`. + let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[1])]); + let stored = BTreeMap::from([ + (shard(0), ShardEpoch(3)), + (shard(1), ShardEpoch(3)), + (shard(2), ShardEpoch(3)), + ]); assert_eq!( - shard_state.raise_epoch_floor(executor(1), &poaching), - vec![shard(0)] + shard_state.raise_epoch_floor_past(&stored), + vec![shard(0), shard(1), shard(2)] + ); + for (shard_id, owner) in [(0, 1), (1, 2)] { + assert_eq!( + shard_state + .shard_assignments + .get(&shard(shard_id)) + .map(|entry| entry.executor_id), + Some(executor(owner)), + "a fenced epoch moved a shard" + ); + assert_eq!( + shard_state.epoch_for_shard(shard(shard_id)), + Some(ShardEpoch(4)) + ); + assert_eq!( + shard_state.shard_epochs.get(&shard(shard_id)), + Some(&ShardEpoch(4)) + ); + } + // An unassigned shard has no entry to put on the rows' epoch: its high-water takes it, and + // the next mint lands one past. + assert_eq!( + shard_state.shard_epochs.get(&shard(2)), + Some(&ShardEpoch(3)) + ); + assert_eq!( + shard_state.next_epoch_for(executor(1), shard(2)), + ShardEpoch(4) ); - assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(9))); assert!(shard_state.check_invariants().is_ok()); + + // Reported again, every epoch is below the record: nothing moves a second time. + assert!(shard_state.raise_epoch_floor_past(&stored).is_empty()); + // At the record is the ordinary loser of a shard move. + assert!( + shard_state + .raise_epoch_floor_past(&BTreeMap::from([(shard(0), ShardEpoch(4))])) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(4))); + assert!(shard_state.check_invariants().is_ok()); + } + + #[test] + fn a_claim_and_a_fenced_epoch_commute_unless_they_are_equal() { + // Executor 1 holds shard 0 at epoch 0. Each case applies a claim on shard 0 and a fenced + // epoch on it, in both orders, to its own copy of that state. + let base = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[1])]); + let epoch_of_shard_0 = |claimant: u128, claimed: u64, fenced: u64, fenced_first: bool| { + let mut shard_state = base.clone(); + let claimed = BTreeMap::from([(shard(0), ShardEpoch(claimed))]); + let fenced = BTreeMap::from([(shard(0), ShardEpoch(fenced))]); + if fenced_first { + shard_state.raise_epoch_floor_past(&fenced); + shard_state.raise_epoch_floor(executor(claimant), &claimed); + } else { + shard_state.raise_epoch_floor(executor(claimant), &claimed); + shard_state.raise_epoch_floor_past(&fenced); + } + assert!(shard_state.check_invariants().is_ok()); + assert_eq!( + shard_state + .shard_assignments + .get(&shard(0)) + .map(|entry| entry.executor_id), + Some(executor(1)) + ); + shard_state + .epoch_for_shard(shard(0)) + .expect("shard 0 is assigned") + }; + + for fenced_first in [false, true] { + // A claim above the fenced epoch wins. + assert_eq!(epoch_of_shard_0(1, 5, 3, fenced_first), ShardEpoch(5)); + // A fenced epoch above the claim ends one past it. + assert_eq!(epoch_of_shard_0(1, 3, 5, fenced_first), ShardEpoch(6)); + // Another executor's claim and an equal fenced epoch both mint the owner one past. + assert_eq!(epoch_of_shard_0(2, 3, 3, fenced_first), ShardEpoch(4)); + } + + // The pair that does not commute: the owner's own claim equal to the fenced epoch. Claim + // first is what happens when the claim reached the state in an earlier request - the + // fenced epoch is then at the record, the equality nothing can tell from a shard move. + assert_eq!(epoch_of_shard_0(1, 3, 3, false), ShardEpoch(3)); + // Fenced first is the order a renewal carrying both uses, which is why it uses it: the + // owner ends one past the rows' epoch rather than on it. + assert_eq!(epoch_of_shard_0(1, 3, 3, true), ShardEpoch(4)); } #[test] diff --git a/golem-shard-manager/src/sharding/shard_management.rs b/golem-shard-manager/src/sharding/shard_management.rs index 78b1deb77f..b87bdaf03c 100644 --- a/golem-shard-manager/src/sharding/shard_management.rs +++ b/golem-shard-manager/src/sharding/shard_management.rs @@ -208,6 +208,19 @@ impl ShardManagement { Ok(shard_management) } + /// Registers the executor instance `executor_id`, listening at `addr`, carrying nothing over + /// from an earlier instance: [`Self::register_executor_with_previous_claim`] with an empty + /// claim. + pub async fn register_executor( + &self, + executor_id: ExecutorId, + addr: ExecutorAddr, + pod_name: Option, + ) -> Result { + self.register_executor_with_previous_claim(executor_id, addr, pod_name, BTreeMap::new()) + .await + } + /// Registers the executor instance `executor_id`, listening at `addr`, and grants it a lease. /// /// The lease is written before this returns, so an acknowledged registration is a durable one: @@ -217,21 +230,53 @@ impl ShardManagement { /// the same registration, so the same id at the same address refreshes that lease and returns /// it; it neither creates a second lease nor counts as a replacement. A *different* id at a /// known address is a restarted instance, and inherits its predecessor's shards. - pub async fn register_executor( + /// + /// `previous_claim` is the set the executor held under an earlier id that was answered + /// [`ShardManagerError::ShardLeaseNotFound`], and is empty on a process's first registration. + /// It is evidence and never assigns a shard. Against a store that kept its history it is at or + /// below the record and moves nothing. A store that was wiped or replaced no longer lists the + /// earlier id, so it refused the renewal that would have repaired it, and this is the moment + /// the executor can tell it what it forgot - see [`ShardLeaseState::raise_epoch_floor`]. An + /// unassigned shard's high-water rises to the claim, so the loop's first mint lands one past + /// it; a shard another executor already holds is re-minted one past the claim, and that owner + /// is pushed its new epoch. + pub async fn register_executor_with_previous_claim( &self, executor_id: ExecutorId, addr: ExecutorAddr, pod_name: Option, + previous_claim: BTreeMap, ) -> Result { - debug!(executor_id = %executor_id, addr = %addr, "Registering executor"); + debug!( + executor_id = %executor_id, + addr = %addr, + previous_shards = previous_claim.len(), + "Registering executor" + ); let now = Utc::now(); let lease_ttl = self.lease_ttl; - let ((already_known, replaced, number_of_shards, pending), stored_at) = self - .persist_for_request(move |shard_state| { + let ((already_known, replaced, number_of_shards, pending, re_minted_owners), stored_at) = + self.persist_for_request(move |shard_state| { let already_known = shard_state.has_executor(executor_id); let replaced = shard_state.add_executor(executor_id, addr, pod_name, now, lease_ttl); + + // After `add_executor`, so a replaced predecessor's shards are already this + // executor's and no owner that is about to disappear is re-minted; ahead of the + // grant, so the grant carries the repaired epochs. + let raised = shard_state.raise_epoch_floor(executor_id, &previous_claim); + let re_minted_owners = owners_re_minted_by(shard_state, &raised, executor_id); + if !raised.is_empty() { + warn!( + executor_id = %executor_id, + raised_shards = raised.iter().join(", "), + re_minted_owners = re_minted_owners.iter().join(", "), + "Registration carried epochs ahead of the stored state; raising them. \ + The shard state has lost history - it was wiped, replaced or restored" + ); + } + let pending = shard_state.lease_grant_for(executor_id).ok_or_else(|| { ShardManagerError::Internal(format!( "executor {executor_id} holds no lease right after being registered" @@ -242,6 +287,7 @@ impl ShardManagement { replaced, shard_state.number_of_shards, pending, + re_minted_owners, )) }) .await?; @@ -264,16 +310,37 @@ impl ShardManagement { self.updates.lock().await.retry_full_assignment(executor_id); } else if already_known { // A retried registration. The lease clock restarts and the same set comes back; the - // shards are not touched, so their epochs do not move. + // shards are not touched, and the claim it carries was already applied by the attempt + // that stored the lease, so their epochs do not move. info!(executor_id = %executor_id, addr = %addr, "Executor lease refreshed"); } else { info!(executor_id = %executor_id, addr = %addr, "Executor added"); } + if !re_minted_owners.is_empty() { + // After the persist, so the pass pushes a stored epoch. The registration's grant does + // not reach these owners, and until they adopt the new epoch the claimant's oplog rows + // refuse their writes; their own renewals could be a third of a lease away. + let mut updates = self.updates.lock().await; + for owner in &re_minted_owners { + updates.retry_full_assignment(*owner); + } + } + self.change.notify_one(); Ok(ack) } + /// [`Self::renew_shard_lease_with_fenced_epochs`] reporting no fenced epochs. + pub async fn renew_shard_lease( + &self, + executor_id: ExecutorId, + claimed: BTreeMap, + ) -> Result { + self.renew_shard_lease_with_fenced_epochs(executor_id, claimed, BTreeMap::new()) + .await + } + /// Extends `executor_id`'s shard lease and returns the manager's set for it. /// /// The claimed shards are what the executor believes it holds. A claim that does not match - @@ -284,21 +351,34 @@ impl ShardManagement { /// that refusal stores nothing: the mutation runs on a clone that is dropped when the closure /// refuses. /// - /// A renewal never advances an epoch: the epoch is an ownership generation, and moving it on a - /// renewal would make a lost response permanently fatal for a shard the executor still owns. + /// A renewal never mints a new epoch against a record that is intact: the epoch is an + /// ownership generation, and moving it on a renewal would make a lost response permanently + /// fatal for a shard the executor still owns. The exception is a store that lost history - + /// see [`ShardLeaseState::raise_epoch_floor`]: a claim ahead of the record restores the + /// claimant's own epochs, and re-mints the owner of another executor's shard one past it. + /// + /// `fenced` is what the oplog writes this executor was refused found on the rows - see + /// [`ShardLeaseState::raise_epoch_floor_past`]. Ahead of the record it re-mints every owner of + /// the shard one past it, this executor included; at or below the record, which is the + /// ordinary loser of a shard move, it moves nothing. It is applied before `claimed`, so a + /// claim equal to a fenced epoch cannot leave this executor on the rows' `(shard, epoch)`. /// /// Leases that have already lapsed are reaped *before* this one is looked up, so an executor /// whose lease expired while its renewal was in flight is told - /// [`ShardManagerError::ShardLeaseNotFound`] rather than silently resurrected. This does not - /// notify the loop; the shards that reaping freed are picked up by the next tick. - pub async fn renew_shard_lease( + /// [`ShardManagerError::ShardLeaseNotFound`] rather than silently resurrected. The loop is + /// notified only when the claim or a fenced epoch re-minted another executor's shard, so that + /// owner is pushed its new epoch instead of waiting for its own renewal; the shards that + /// reaping freed are picked up by the next tick. + pub async fn renew_shard_lease_with_fenced_epochs( &self, executor_id: ExecutorId, claimed: BTreeMap, + fenced: BTreeMap, ) -> Result { debug!( executor_id = %executor_id, claimed_shards = claimed.len(), + fenced_shards = fenced.len(), "Renewing shard lease" ); let now = Utc::now(); @@ -321,7 +401,7 @@ impl ShardManagement { }) .await?; - let (pending, stored_at) = self + let ((pending, re_minted_owners), stored_at) = self .persist_for_request(move |shard_state| { if !shard_state.has_executor(executor_id) { return Err(ShardManagerError::ShardLeaseNotFound { executor_id }); @@ -353,17 +433,48 @@ impl ShardManagement { ); } - // Ahead of the renewal, so the grant read below carries the repaired epochs. This - // only ever fires when the stored state is behind the cluster it is managing. + // Both repairs run ahead of the renewal, so the grant read below carries the repaired + // epochs, and both only ever fire when the stored state is behind the cluster it is + // managing. + // + // The fenced epochs go first. A fence only says somebody wrote rows at that epoch while + // this executor asserted a lower one. If this request's claim equals it, applying the + // claim first would record it and leave this executor on the rows' `(shard, epoch)`; + // applied first, a fenced epoch at or above the claim ends one past it, and a higher + // claim still wins. They reach a state that was wiped or replaced as well, because the + // executor keeps reporting them until a renewal under its re-registered id is granted. + let re_minted = shard_state.raise_epoch_floor_past(&fenced); + let fence_re_minted_owners = owners_re_minted_by(shard_state, &re_minted, executor_id); + if !re_minted.is_empty() { + warn!( + executor_id = %executor_id, + re_minted_shards = re_minted.iter().join(", "), + re_minted_owners = fence_re_minted_owners.iter().join(", "), + "Fenced oplog writes reported epochs ahead of the stored state; re-minting above \ + them. The shard state has lost history - it was wiped, replaced or restored" + ); + } + + // A claim ahead of the record reaches only a state that still lists this executor: one + // that was wiped or replaced refused the renewal above, and is repaired by the + // re-registration that follows it. let raised = shard_state.raise_epoch_floor(executor_id, &claimed); + let claim_re_minted_owners = owners_re_minted_by(shard_state, &raised, executor_id); if !raised.is_empty() { warn!( executor_id = %executor_id, raised_shards = raised.iter().join(", "), + re_minted_owners = claim_re_minted_owners.iter().join(", "), "Shard lease claim carried epochs ahead of the stored state; raising them. \ - The shard state has lost history - it was wiped, restored or replaced" + The shard state has lost history - it was restored from a backup" ); } + // The moved shards another executor owns were re-minted. This renewal's grant does not + // reach that owner, so it has to be pushed the new epoch. + let re_minted_owners: BTreeSet = fence_re_minted_owners + .union(&claim_re_minted_owners) + .copied() + .collect(); if !shard_state.renew_lease(executor_id, now, lease_ttl) { return Err(ShardManagerError::Internal(format!( @@ -373,13 +484,30 @@ impl ShardManagement { // Read off the mutated clone, so the grant this returns is exactly the state that is // about to be stored - never a state that a failed write then rolls back. - shard_state.lease_grant_for(executor_id).ok_or_else(|| { - ShardManagerError::Internal(format!( - "executor {executor_id} holds no lease right after it was renewed" - )) - }) + shard_state + .lease_grant_for(executor_id) + .map(|pending| (pending, re_minted_owners)) + .ok_or_else(|| { + ShardManagerError::Internal(format!( + "executor {executor_id} holds no lease right after it was renewed" + )) + }) }) .await?; + + if !re_minted_owners.is_empty() { + // After the persist, so the pass pushes a stored epoch. Until the owner adopts it, its + // writes on the re-minted shards carry the epoch the store forgot and the claimant's + // oplog rows refuse them; its own renewal could be a third of a lease away. + { + let mut updates = self.updates.lock().await; + for owner in &re_minted_owners { + updates.retry_full_assignment(*owner); + } + } + self.change.notify_one(); + } + // Read off the clone before its revision was bumped; stamped with the revision the state // was then stored at, so the grant names exactly the persisted state it describes. Ok(pending.stamp(stored_at)) @@ -465,9 +593,10 @@ impl ShardManagement { threshold: f64, ) -> Result<(), ShardManagerError> { // The timer is what makes the pull-based half of the lease protocol work. `RenewShardLease` - // and `Deregister` never wake the loop - they only leave shards behind - so without a tick - // an expired lease in a quiet cluster would never be reaped and a graceful shutdown's - // shards would never be re-homed. A third of the lease is the same cadence the executors + // and `Deregister` only leave shards behind - the one wake-up is a renewal whose claim + // re-minted another executor's shard after the store lost history, which the loop must + // push to that owner - so without a tick an expired lease in a quiet cluster would never be + // reaped and a graceful shutdown's shards would never be re-homed. A third of the lease is the same cadence the executors // renew at, and it is derived rather than configured so there is no second knob to keep // consistent with the lease duration. let tick_period = shard_lease::renewal_interval(self.lease_ttl); @@ -853,6 +982,22 @@ impl ShardManagement { } } +/// The executors other than `claimant` that own a shard in `raised`: the owners +/// [`ShardLeaseState::raise_epoch_floor`] re-minted one past the claim. The claimant's grant does +/// not reach them, so each is owed a push of its new epoch. +fn owners_re_minted_by( + shard_state: &ShardLeaseState, + raised: &[ShardId], + claimant: ExecutorId, +) -> BTreeSet { + raised + .iter() + .filter_map(|shard_id| shard_state.shard_assignments.get(shard_id)) + .map(|entry| entry.executor_id) + .filter(|owner| *owner != claimant) + .collect() +} + /// The full-replace payloads for `executor_ids`, read off `shard_state`. /// /// The one place a push is built, so the never-zero `number_of_shards` guard has a single home. diff --git a/golem-shard-manager/tests/shard_management.rs b/golem-shard-manager/tests/shard_management.rs index e34db60f36..6ec6982c1a 100644 --- a/golem-shard-manager/tests/shard_management.rs +++ b/golem-shard-manager/tests/shard_management.rs @@ -1256,6 +1256,184 @@ async fn a_repeated_registration_of_the_same_executor_refreshes_its_lease() { join_set.abort_all(); } +#[test] +// A store that was wiped or replaced no longer lists the executor, so its renewal is refused as a +// lease not found and never reaches the repair a renewal makes. The executor re-registers under a +// fresh id carrying the set it held, and the oplog rows it wrote are fenced at those epochs: the +// manager has to mint above them, or every write to those agents is refused for good. +async fn a_re_registration_after_a_wiped_store_mints_above_the_epochs_it_held() { + let restarted_pod = pod(1, 9000); + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(ShardLeaseState::new(4), worker_executors.clone()).await; + + let held = BTreeMap::from([ + (ShardId::new(0), ShardEpoch(2)), + (ShardId::new(1), ShardEpoch(2)), + (ShardId::new(2), ShardEpoch(5)), + (ShardId::new(3), ShardEpoch(2)), + ]); + let ack = shard_management + .register_executor_with_previous_claim(executor(1), restarted_pod.into(), None, held) + .await + .expect("the registration should have been persisted"); + // Evidence, not a request: nothing is assigned until the loop's pass. + assert!(ack.grant.shard_epochs.is_empty()); + + // The pass mints after the registration was stored, so it is the barrier for the epochs. + wait_for_local_assignment(&worker_executors, restarted_pod, shard_ids(&[0, 1, 2, 3])).await; + wait_for_quiescence(&persistence).await; + + let minted = BTreeMap::from([ + (ShardId::new(0), ShardEpoch(3)), + (ShardId::new(1), ShardEpoch(3)), + (ShardId::new(2), ShardEpoch(6)), + (ShardId::new(3), ShardEpoch(3)), + ]); + let after = persistence.latest().await; + assert_eq!( + claim_of(&after, executor(1)), + minted, + "the pass minted from a floor the store forgot" + ); + assert!(after.check_invariants().is_ok()); + + let pushed = worker_executors + .pushes_to(restarted_pod) + .await + .pop() + .expect("the registered executor should have been pushed its set"); + assert_eq!(pushed.shard_epochs, minted); + + join_set.abort_all(); +} + +#[test] +// The same carried set against a store that kept its history is at or below the record, so it moves +// nothing: the shards a re-registered executor is given are minted exactly as they are for a +// registration that carried nothing, one past the record. +async fn a_re_registration_against_an_intact_store_mints_as_if_it_carried_nothing() { + let restarted_pod = pod(1, 9000); + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + + let before = persistence.latest().await; + let held = claim_of(&before, executor(1)); + // The lease is gone, as it is once reaped, so the executor comes back under a fresh id - with + // nothing applied locally, as after it cleared its assignment. + shard_management + .deregister_executor(executor(1), held.clone()) + .await + .expect("the deregistration should have been persisted"); + worker_executors + .set_local_assignment(restarted_pod, &[]) + .await; + + let ack = shard_management + .register_executor_with_previous_claim( + executor(3), + restarted_pod.into(), + Some("worker-executor-0".to_string()), + held, + ) + .await + .expect("the registration should have been persisted"); + assert!(ack.grant.shard_epochs.is_empty()); + + wait_for_local_assignment(&worker_executors, restarted_pod, shard_ids(&[0, 1])).await; + wait_for_quiescence(&persistence).await; + + let after = persistence.latest().await; + assert_eq!( + claim_of(&after, executor(3)), + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(1)), + (ShardId::new(1), ShardEpoch(1)), + ]), + "a carried set at the record moved an epoch" + ); + assert_eq!( + claim_of(&after, executor(2)), + claim_of(&before, executor(2)) + ); + assert!(after.check_invariants().is_ok()); + + join_set.abort_all(); +} + +#[test] +// A re-registration's carried set can name a shard the manager has given to another executor. Ahead +// of the record, it proves the claimant's oplog rows for that shard are fenced above the epoch the +// owner holds, so the owner is re-minted one past the claim and pushed that epoch at once: the +// registration's grant does not reach it, and its own renewal could be a third of a lease away. +async fn a_re_registration_carrying_a_claim_on_another_executors_shard_pushes_its_owner() { + let restarted_pod = pod(1, 9000); + let owner_pod = pod(2, 9001); + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + assert_eq!(worker_executors.pushes_to(restarted_pod).await.len(), 1); + assert_eq!(worker_executors.pushes_to(owner_pod).await.len(), 1); + + let before = persistence.latest().await; + let mut held = claim_of(&before, executor(1)); + held.insert(ShardId::new(2), ShardEpoch(5)); + let ack = shard_management + .register_executor_with_previous_claim( + executor(3), + restarted_pod.into(), + Some("worker-executor-0".to_string()), + held, + ) + .await + .expect("the registration should have been persisted"); + // A new instance at a known address inherits its predecessor's shards one past the record; the + // claim on the owner's shard does not hand that shard over. + assert_eq!( + ack.grant.shard_epochs, + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(1)), + (ShardId::new(1), ShardEpoch(1)), + ]) + ); + + let stored = persistence + .state_at(ack.grant.revision) + .await + .expect("the ack must name a revision the store really held"); + assert_eq!( + stored.shards_for_executor(executor(2)), + Some(shard_ids(&[2, 3])) + ); + assert_eq!(stored.epoch_for_shard(ShardId::new(2)), Some(ShardEpoch(6))); + assert_eq!(stored.epoch_for_shard(ShardId::new(3)), Some(ShardEpoch(0))); + assert!(stored.check_invariants().is_ok()); + + // Startup pushed once to each executor and the replacement pushes the restarted one, so the + // owner's push is the fourth. The tick is a third of a 60s lease away, so only the registration + // queueing the owner can send it within the wait. + wait_for_pushes(&worker_executors, 4).await; + wait_for_quiescence(&persistence).await; + + let owner_pushes = worker_executors.pushes_to(owner_pod).await; + assert_eq!(owner_pushes.len(), 2); + let pushed = owner_pushes.last().expect("the owner was pushed"); + assert_eq!( + pushed.shard_epochs, + BTreeMap::from([ + (ShardId::new(2), ShardEpoch(6)), + (ShardId::new(3), ShardEpoch(0)), + ]) + ); + assert!( + pushed.revision >= ack.grant.revision, + "the push was read off a state older than the re-mint" + ); + + join_set.abort_all(); +} + #[test] // The loop ends with the error rather than carry on against a store it can no longer trust, so // the process restarts and re-reads. On a conflict the cached revision is deliberately not @@ -2020,9 +2198,10 @@ async fn renewing_twice_with_the_same_epochs_moves_nothing() { } #[test] -// The claim is what the executor believes it holds, not a condition of the renewal. A wrong epoch, -// another executor's shard and a released shard are all an executor that missed a push, and -// refusing its renewal would only hold it on a picture the manager knows is wrong. It is renewed, +// The claim is what the executor believes it holds, not a condition of the renewal. Another +// executor's shard and a released shard are an executor that missed a push, and an epoch ahead of +// the record on another executor's shard is one the manager forgot; refusing either renewal would +// only hold the executor on a picture the manager knows is wrong. It is renewed, // and the grant carries the manager's set: the renewal is the guaranteed second delivery path for a // push that was lost. Only an executor the manager has never heard of is refused. async fn a_mismatched_claim_is_renewed_and_corrected() { @@ -2046,27 +2225,50 @@ async fn a_mismatched_claim_is_renewed_and_corrected() { "got {err:?}" ); - // a wrong epoch on a shard that belongs to somebody else, alongside claim entries that are - // perfectly valid. Corrected and never adopted: an epoch stamped onto another executor's - // assignment would leave both of them live on one `(shard, epoch)`, which is the one pair the - // oplog fence cannot separate. - let mut wrong_epoch = truth.clone(); - wrong_epoch.insert(ShardId::new(2), ShardEpoch(7)); + // an epoch ahead of the record on a shard that belongs to somebody else, alongside claim + // entries that are perfectly valid. Corrected and never adopted: an epoch stamped onto another + // executor's assignment would leave both of them live on one `(shard, epoch)`, which is the one + // pair the oplog fence cannot separate. Being ahead, the claim proves the store lost history, so + // the owner keeps the shard and is re-minted one past the claim, and the claimant is not given + // it. + let mut ahead_of_owner = truth.clone(); + ahead_of_owner.insert(ShardId::new(2), ShardEpoch(7)); let expiry_before = expiry_of(&persistence.latest().await, executor(1)); let grant = shard_management - .renew_shard_lease(executor(1), wrong_epoch) + .renew_shard_lease(executor(1), ahead_of_owner) .await - .expect("a claim at the wrong epoch is renewed and corrected"); + .expect("a claim ahead of another executor's shard is renewed and corrected"); assert_eq!( grant.shard_epochs, truth, "the grant is the manager's set, not the claim" ); assert!(grant.expires_at > expiry_before, "the lease was extended"); + + // The re-mint woke the loop to push the owner its new epoch, and that pass has to finish before + // executor 2 deregisters below, or it could re-home the shards the deregistration releases. + // Startup pushed once to each executor, so the owner's push is the third. + wait_for_pushes(&worker_executors, 3).await; + wait_for_quiescence(&persistence).await; + let re_minted = persistence.latest().await; + assert_eq!( + re_minted.epoch_for_shard(ShardId::new(2)), + Some(ShardEpoch(8)), + "the owner was not re-minted one past the claim" + ); assert_eq!( - persistence.latest().await.epoch_for_shard(ShardId::new(2)), - Some(ShardEpoch(0)), + re_minted.shards_for_executor(executor(2)), + Some(shard_ids(&[2, 3])), "the claim moved another executor's shard" ); + assert_eq!( + worker_executors + .pushes_to(pod(2, 9001)) + .await + .last() + .and_then(|push| push.shard_epochs.get(&ShardId::new(2)).copied()), + Some(ShardEpoch(8)), + "the owner was not pushed its re-minted epoch" + ); // a shard that belongs to another executor let moved = BTreeMap::from([(ShardId::new(2), ShardEpoch(0))]); @@ -2092,12 +2294,14 @@ async fn a_mismatched_claim_is_renewed_and_corrected() { .expect("claiming a released shard is renewed and corrected"); assert_eq!(grant.shard_epochs, truth); - // None of the corrections moved anything: executor 1's set and epochs are exactly what they - // were, only its lease clock moved, and executor 2's released shards stayed released. + // No correction moved executor 1's set or epochs: they are exactly what they were, only its + // lease clock moved, and executor 2's released shards stayed released. The one thing that + // moved was shard 2's epoch, and its release keeps it as the floor the next owner mints above. let after = persistence.latest().await; assert_eq!(claim_of(&after, executor(1)), truth); assert!(expiry_of(&after, executor(1)) > expiry_before); assert_eq!(after.get_unassigned_shards(), shard_ids(&[2, 3])); + assert_eq!(after.shard_epochs[&ShardId::new(2)], ShardEpoch(8)); join_set.abort_all(); } @@ -2145,6 +2349,324 @@ async fn a_claim_ahead_of_the_record_raises_the_managers_floor() { join_set.abort_all(); } +#[test] +// A claim ahead of the record on a shard the manager has since given to another executor. The store +// lost history, so the claimant's oplog rows for that shard are fenced above the epoch the owner +// holds, and every write the owner makes is refused. The owner keeps the shard and is minted one +// past the claim - never onto it, which would put two live executors on one `(shard, epoch)` - and +// is pushed that epoch at once rather than left to find out on its own renewal. +async fn a_higher_claim_on_another_executors_shard_re_mints_and_pushes_its_owner() { + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + assert_eq!(worker_executors.pushes_to(pod(1, 9000)).await.len(), 1); + assert_eq!(worker_executors.pushes_to(pod(2, 9001)).await.len(), 1); + + let before = persistence.latest().await; + let mut ahead_of_owner = claim_of(&before, executor(1)); + ahead_of_owner.insert(ShardId::new(2), ShardEpoch(7)); + let grant = shard_management + .renew_shard_lease(executor(1), ahead_of_owner) + .await + .expect("a claim ahead of another executor's shard is renewed, not refused"); + assert_eq!( + grant.shard_epochs, + claim_of(&before, executor(1)), + "the claimant was given another executor's shard" + ); + + let after = persistence.latest().await; + assert_eq!( + after.shards_for_executor(executor(2)), + Some(shard_ids(&[2, 3])) + ); + assert_eq!(after.epoch_for_shard(ShardId::new(2)), Some(ShardEpoch(8))); + assert_eq!(after.epoch_for_shard(ShardId::new(3)), Some(ShardEpoch(0))); + assert_eq!( + claim_of(&after, executor(1)), + claim_of(&before, executor(1)) + ); + assert!(after.check_invariants().is_ok()); + + // Startup pushed once to each executor, so the owner's push is the third. The tick is a third of + // a 60s lease away, so only the renewal waking the loop can send it within the wait. + wait_for_pushes(&worker_executors, 3).await; + wait_for_quiescence(&persistence).await; + + let owner_pushes = worker_executors.pushes_to(pod(2, 9001)).await; + assert_eq!(owner_pushes.len(), 2); + let pushed = owner_pushes.last().expect("the owner was pushed"); + assert_eq!( + pushed.shard_epochs, + BTreeMap::from([ + (ShardId::new(2), ShardEpoch(8)), + (ShardId::new(3), ShardEpoch(0)), + ]) + ); + assert!( + pushed.revision >= grant.revision, + "the push was read off a state older than the re-mint" + ); + assert_eq!( + worker_executors.pushes_to(pod(1, 9000)).await.len(), + 1, + "the claimant was pushed as well" + ); + + join_set.abort_all(); +} + +#[test] +// A renewal can carry the epoch an executor's refused oplog write found on the rows. Above the +// record it proves the store lost history, but not that the reporter wrote those rows, so even the +// reporter's own shard is minted one past that epoch rather than onto it. When the same renewal also +// claims that epoch, the fenced epoch is applied first: claim first would record the claim and +// leave the reporter on the rows' `(shard, epoch)`. +async fn a_renewal_reporting_a_fenced_epoch_re_mints_the_reporters_own_shard_above_it() { + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + + let before = persistence.latest().await; + let grant = shard_management + .renew_shard_lease_with_fenced_epochs( + executor(1), + claim_of(&before, executor(1)), + BTreeMap::from([(ShardId::new(1), ShardEpoch(3))]), + ) + .await + .expect("a renewal reporting a fenced epoch is renewed, not refused"); + assert_eq!( + grant.shard_epochs, + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(0)), + (ShardId::new(1), ShardEpoch(4)), + ]), + "the grant does not carry the re-minted epoch" + ); + + let after = persistence.latest().await; + assert_eq!(after.epoch_for_shard(ShardId::new(1)), Some(ShardEpoch(4))); + assert_eq!( + after.shards_for_executor(executor(1)), + Some(shard_ids(&[0, 1])), + "re-minting an epoch moved a shard" + ); + assert_eq!( + claim_of(&after, executor(2)), + claim_of(&before, executor(2)), + "a fenced epoch on one executor's shard disturbed another executor" + ); + assert!(after.check_invariants().is_ok()); + + // The reporter learns its re-minted epoch from the grant, so nobody is owed a push. + wait_for_quiescence(&persistence).await; + assert_eq!(worker_executors.pushes_to(pod(1, 9000)).await.len(), 1); + assert_eq!(worker_executors.pushes_to(pod(2, 9001)).await.len(), 1); + join_set.abort_all(); + + // The same epoch claimed and reported in one request. + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + let grant = shard_management + .renew_shard_lease_with_fenced_epochs( + executor(1), + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(0)), + (ShardId::new(1), ShardEpoch(3)), + ]), + BTreeMap::from([(ShardId::new(1), ShardEpoch(3))]), + ) + .await + .expect("a renewal claiming and reporting one epoch is renewed, not refused"); + assert_eq!( + grant.shard_epochs, + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(0)), + (ShardId::new(1), ShardEpoch(4)), + ]), + "the claim was applied before the fenced epoch, leaving the executor on the rows' epoch" + ); + assert_eq!( + persistence.latest().await.epoch_for_shard(ShardId::new(1)), + Some(ShardEpoch(4)) + ); + + join_set.abort_all(); +} + +#[test] +// A fenced epoch on a shard the manager has given to another executor: the reporter lost the shard +// and was refused by rows above the owner's recorded epoch, which only a store that lost history can +// produce. The owner keeps the shard, is minted one past the rows, and is pushed that epoch at once +// rather than left to find out on its own renewal; the reporter is not given the shard. +async fn a_fenced_epoch_reported_by_a_non_owner_re_mints_and_pushes_the_owner() { + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + + let before = persistence.latest().await; + let grant = shard_management + .renew_shard_lease_with_fenced_epochs( + executor(1), + claim_of(&before, executor(1)), + BTreeMap::from([(ShardId::new(2), ShardEpoch(7))]), + ) + .await + .expect("a renewal reporting a fenced epoch on another executor's shard is renewed"); + assert_eq!( + grant.shard_epochs, + claim_of(&before, executor(1)), + "the reporter was given another executor's shard" + ); + + let after = persistence.latest().await; + assert_eq!( + after.shards_for_executor(executor(2)), + Some(shard_ids(&[2, 3])) + ); + assert_eq!(after.epoch_for_shard(ShardId::new(2)), Some(ShardEpoch(8))); + assert_eq!(after.epoch_for_shard(ShardId::new(3)), Some(ShardEpoch(0))); + assert!(after.check_invariants().is_ok()); + + // Startup pushed once to each executor, so the owner's push is the third. The tick is a third + // of a 60s lease away, so only the renewal waking the loop can send it within the wait. + wait_for_pushes(&worker_executors, 3).await; + wait_for_quiescence(&persistence).await; + + let owner_pushes = worker_executors.pushes_to(pod(2, 9001)).await; + assert_eq!(owner_pushes.len(), 2); + let pushed = owner_pushes.last().expect("the owner was pushed"); + assert_eq!( + pushed.shard_epochs, + BTreeMap::from([ + (ShardId::new(2), ShardEpoch(8)), + (ShardId::new(3), ShardEpoch(0)), + ]) + ); + assert!( + pushed.revision >= grant.revision, + "the push was read off a state older than the re-mint" + ); + assert_eq!( + worker_executors.pushes_to(pod(1, 9000)).await.len(), + 1, + "the reporter was pushed as well" + ); + + join_set.abort_all(); +} + +#[test] +// A fenced epoch at or below the record is the ordinary loser of a shard move - the new owner +// recorded its epoch, and the old one was refused - and a report the manager already applied is at +// the record the second time. Neither moves an epoch or owes anyone a push. A renewal the manager +// refuses applies nothing it carried. +async fn a_fenced_epoch_at_or_below_the_record_moves_nothing() { + let worker_executors = Arc::new(TestWorkerExecutors::default()); + let (shard_management, persistence, mut join_set) = + new_shard_management(balanced_pair(), worker_executors.clone()).await; + + // A restarted instance at executor 1's address inherits its shards at epoch 1, so executor + // 1's writes at epoch 0 are what the rows refuse from now on. + shard_management + .register_executor( + executor(3), + pod(1, 9000).into(), + Some("worker-executor-0".to_string()), + ) + .await + .expect("the registration should have been persisted"); + wait_for_pushes(&worker_executors, 3).await; + wait_for_quiescence(&persistence).await; + let moved = persistence.latest().await; + assert_eq!( + claim_of(&moved, executor(3)), + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(1)), + (ShardId::new(1), ShardEpoch(1)), + ]) + ); + let pushes = worker_executors.pushes.lock().await.len(); + + let grant = shard_management + .renew_shard_lease_with_fenced_epochs( + executor(2), + claim_of(&moved, executor(2)), + BTreeMap::from([ + (ShardId::new(0), ShardEpoch(1)), + (ShardId::new(1), ShardEpoch(0)), + ]), + ) + .await + .expect("a renewal reporting a fenced epoch at the record is renewed"); + assert_eq!(grant.shard_epochs, claim_of(&moved, executor(2))); + let after = persistence.latest().await; + assert_eq!(after.shard_epochs, moved.shard_epochs); + assert_eq!(after.shard_assignments, moved.shard_assignments); + wait_for_quiescence(&persistence).await; + assert_eq!( + worker_executors.pushes.lock().await.len(), + pushes, + "a fenced epoch at the record owed somebody a push" + ); + + // A genuine report re-mints shard 1's owner once. The same report on the next renewal is at the + // record, so the state stays exactly as the first one left it. + let genuine = BTreeMap::from([(ShardId::new(1), ShardEpoch(5))]); + shard_management + .renew_shard_lease_with_fenced_epochs( + executor(2), + claim_of(&moved, executor(2)), + genuine.clone(), + ) + .await + .expect("a renewal reporting a fenced epoch ahead of the record is renewed"); + wait_for_pushes(&worker_executors, pushes + 1).await; + wait_for_quiescence(&persistence).await; + let raised = persistence.latest().await; + assert_eq!(raised.epoch_for_shard(ShardId::new(1)), Some(ShardEpoch(6))); + let pushes = worker_executors.pushes.lock().await.len(); + + shard_management + .renew_shard_lease_with_fenced_epochs(executor(2), claim_of(&raised, executor(2)), genuine) + .await + .expect("a repeated report is renewed"); + wait_for_quiescence(&persistence).await; + let repeated = persistence.latest().await; + assert_eq!(repeated.shard_epochs, raised.shard_epochs); + assert_eq!(repeated.shard_assignments, raised.shard_assignments); + assert_eq!( + worker_executors.pushes.lock().await.len(), + pushes, + "a repeated report re-minted its owner a second time" + ); + + let err = shard_management + .renew_shard_lease_with_fenced_epochs( + executor(9), + BTreeMap::new(), + BTreeMap::from([(ShardId::new(0), ShardEpoch(9))]), + ) + .await + .expect_err("an unknown executor holds no lease to renew"); + assert!( + matches!( + err, + ShardManagerError::ShardLeaseNotFound { executor_id } if executor_id == executor(9) + ), + "got {err:?}" + ); + assert_eq!( + persistence.latest().await.epoch_for_shard(ShardId::new(0)), + Some(ShardEpoch(1)), + "a refused renewal applied the fenced epoch it carried" + ); + + join_set.abort_all(); +} + #[test] // Every delivery carries the revision of the persisted state that holds its set, so the executor // can order a push and a renewal response that cross on the network. The grant is read off the @@ -2432,9 +2954,9 @@ async fn the_loop_compacts_after_each_pass_and_carries_on_when_compaction_fails( } #[test] -// The lease paths never wake the loop, so the timer is the only thing that can notice an expiry in -// a cluster where nothing else is happening. Without it a lapsed lease is held forever and its -// shards are never re-homed. +// The lease paths wake the loop only to push an owner a renewal re-minted, so the timer is the only +// thing that can notice an expiry in a cluster where nothing else is happening. Without it a lapsed +// lease is held forever and its shards are never re-homed. async fn an_expired_lease_is_reclaimed_within_one_tick() { let worker_executors = Arc::new(TestWorkerExecutors::default()); let (_shard_management, persistence, mut join_set) = start_shard_management( diff --git a/golem-test-framework/src/components/shard_manager/mod.rs b/golem-test-framework/src/components/shard_manager/mod.rs index 6a4ee78ab5..565043fe46 100644 --- a/golem-test-framework/src/components/shard_manager/mod.rs +++ b/golem-test-framework/src/components/shard_manager/mod.rs @@ -88,7 +88,6 @@ async fn wait_for_startup( async fn env_vars( number_of_shards_override: Option, - shard_lease_duration_override: Option, http_port: u16, grpc_port: u16, rdb: &Arc, @@ -129,12 +128,5 @@ async fn env_vars( builder = builder.with("GOLEM__NUMBER_OF_SHARDS", number_of_shards.to_string()); } - if let Some(shard_lease_duration) = shard_lease_duration_override { - builder = builder.with( - "GOLEM__SHARD_LEASE_DURATION", - format!("{}ms", shard_lease_duration.as_millis()), - ); - } - builder.build() } diff --git a/golem-test-framework/src/components/shard_manager/spawned.rs b/golem-test-framework/src/components/shard_manager/spawned.rs index 6b5e47ba0f..e0edef24e7 100644 --- a/golem-test-framework/src/components/shard_manager/spawned.rs +++ b/golem-test-framework/src/components/shard_manager/spawned.rs @@ -29,7 +29,6 @@ pub struct SpawnedShardManager { http_port: u16, grpc_port: u16, number_of_shards_override: std::sync::RwLock>, - shard_lease_duration_override: Option, child: Arc>>, logger: Arc>>, executable: PathBuf, @@ -47,7 +46,6 @@ impl SpawnedShardManager { executable: &Path, working_directory: &Path, number_of_shards_override: Option, - shard_lease_duration_override: Option, http_port: u16, grpc_port: u16, rdb: Arc, @@ -67,7 +65,6 @@ impl SpawnedShardManager { executable, working_directory, number_of_shards_override, - shard_lease_duration_override, http_port, grpc_port, &rdb, @@ -83,7 +80,6 @@ impl SpawnedShardManager { http_port, grpc_port, number_of_shards_override: std::sync::RwLock::new(number_of_shards_override), - shard_lease_duration_override, child: Arc::new(Mutex::new(Some(child))), logger: Arc::new(Mutex::new(Some(logger))), executable: executable.to_path_buf(), @@ -101,7 +97,6 @@ impl SpawnedShardManager { executable: &Path, working_directory: &Path, number_of_shards_override: Option, - shard_lease_duration_override: Option, http_port: u16, grpc_port: u16, rdb: &Arc, @@ -116,7 +111,6 @@ impl SpawnedShardManager { .envs( super::env_vars( number_of_shards_override, - shard_lease_duration_override, http_port, grpc_port, rdb, @@ -187,7 +181,6 @@ impl ShardManager for SpawnedShardManager { &self.executable, &self.working_directory, number_of_shards_override, - self.shard_lease_duration_override, self.http_port, self.grpc_port, &self.rdb, diff --git a/golem-test-framework/src/components/worker_executor/spawned.rs b/golem-test-framework/src/components/worker_executor/spawned.rs index e6afd96f69..d05eb30ec3 100644 --- a/golem-test-framework/src/components/worker_executor/spawned.rs +++ b/golem-test-framework/src/components/worker_executor/spawned.rs @@ -188,27 +188,50 @@ impl SpawnedWorkerExecutor { #[cfg(unix)] fn signal_child(&self, signal: libc::c_int, action: &str) { - let child = self.child.lock().unwrap(); - let child = child.as_ref().unwrap_or_else(|| { + // The guard is held across the liveness check and the signal: `is_running` and + // `blocking_kill`, the only other reapers of this child, both take the same lock. + let mut child_field = self.child.lock().unwrap(); + let child = child_field.as_mut().unwrap_or_else(|| { panic!( "Cannot {action} golem-worker-executor {}: it is not running", self.grpc_port ) }); - let pid = libc::pid_t::try_from(child.id()).expect("child pid does not fit into pid_t"); - // SAFETY: `kill` has no memory-safety preconditions. The pid belongs to a child this struct - // spawned and has not reaped, so it cannot have been reused by another process. - let result = unsafe { libc::kill(pid, signal) }; - assert_eq!( - result, - 0, - "Failed to {action} golem-worker-executor {}: {}", - self.grpc_port, - std::io::Error::last_os_error() + signal_unreaped_child( + child, + signal, + &format!("{action} golem-worker-executor {}", self.grpc_port), ); } } +/// Sends `signal` to `child`, refusing to if the child has already been reaped. `what` names the +/// action and the process for the panic messages. +/// +/// A reaped child's pid is free for the OS to hand to an unrelated process, and `is_running`'s +/// `try_wait` reaps an exited child while leaving it in place, so a raw `kill` on `child.id()` +/// alone could signal a stranger. `Child::kill` has this guard built in; `kill(2)` does not. +#[cfg(unix)] +fn signal_unreaped_child(child: &mut Child, signal: libc::c_int, what: &str) { + match child.try_wait() { + Ok(None) => {} + Ok(Some(status)) => panic!("Cannot {what}: it has already exited ({status})"), + Err(err) => panic!("Cannot {what}: its state is unknown: {err}"), + } + let pid = libc::pid_t::try_from(child.id()).expect("child pid does not fit into pid_t"); + // SAFETY: `kill` has no memory-safety preconditions. `try_wait` has just reported the child + // alive, and reaping it needs the `&mut Child` held here, so it has not been reaped and its pid + // cannot belong to another process. If it exited since, it is a zombie still holding that pid, + // and the signal changes nothing. + let result = unsafe { libc::kill(pid, signal) }; + assert_eq!( + result, + 0, + "Failed to {what}: {}", + std::io::Error::last_os_error() + ); +} + #[async_trait] impl WorkerExecutor for SpawnedWorkerExecutor { fn grpc_host(&self) -> String { @@ -291,6 +314,24 @@ impl WorkerExecutor for SpawnedWorkerExecutor { info!("Resuming golem-worker-executor {}", self.grpc_port); self.signal_child(libc::SIGCONT, "resume"); } + + // Without these the trait default would panic claiming this is not a SpawnedWorkerExecutor, + // when the real reason is the platform. + #[cfg(not(unix))] + async fn pause(&self) { + panic!( + "Cannot pause golem-worker-executor {}: pausing is SIGSTOP, which this platform does not have", + self.grpc_port + ); + } + + #[cfg(not(unix))] + async fn resume(&self) { + panic!( + "Cannot resume golem-worker-executor {}: resuming is SIGCONT, which this platform does not have", + self.grpc_port + ); + } } impl Drop for SpawnedWorkerExecutor { @@ -298,3 +339,48 @@ impl Drop for SpawnedWorkerExecutor { self.blocking_kill(); } } + +#[cfg(all(test, unix))] +mod tests { + use test_r::test; + + use super::signal_unreaped_child; + use std::process::{Child, Command}; + + /// Kills and reaps the child when the test ends, also when an assertion panics, so a failing + /// run does not leave a stopped process behind. + struct KilledOnDrop(Child); + + impl Drop for KilledOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[test] + #[should_panic(expected = "has already exited")] + fn a_child_that_has_been_reaped_is_never_signalled() { + let mut child = Command::new("true") + .spawn() + .expect("failed to spawn `true`"); + // Records the exit status, exactly as `is_running`'s `try_wait` does for an exited child. + child.wait().expect("failed to wait for `true`"); + + signal_unreaped_child(&mut child, libc::SIGSTOP, "pause `true`"); + } + + #[test] + fn a_live_child_can_be_stopped_and_continued() { + let mut child = KilledOnDrop( + Command::new("sleep") + .arg("30") + .spawn() + .expect("failed to spawn `sleep`"), + ); + + signal_unreaped_child(&mut child.0, libc::SIGSTOP, "pause `sleep`"); + // `try_wait` does not report a stopped child, so continuing it is not refused as exited. + signal_unreaped_child(&mut child.0, libc::SIGCONT, "resume `sleep`"); + } +} diff --git a/golem-test-framework/src/config/benchmark.rs b/golem-test-framework/src/config/benchmark.rs index 61aacb58ad..864f215990 100644 --- a/golem-test-framework/src/config/benchmark.rs +++ b/golem-test-framework/src/config/benchmark.rs @@ -430,7 +430,6 @@ impl BenchmarkTestDependencies { &build_root.join("golem-shard-manager"), &workspace_root.join("golem-shard-manager"), None, - None, shard_manager_http_port, shard_manager_grpc_port, rdb.clone(), diff --git a/golem-test-framework/src/config/env.rs b/golem-test-framework/src/config/env.rs index e53071c2a1..e84f32ba1e 100644 --- a/golem-test-framework/src/config/env.rs +++ b/golem-test-framework/src/config/env.rs @@ -93,9 +93,6 @@ pub struct EnvBasedTestDependenciesConfig { pub worker_executor_cluster_size: usize, pub environment_state_cache_capacity: Option, pub number_of_shards_override: Option, - /// The shard manager's `shard_lease_duration`, for tests that have to watch a lease lapse - /// without waiting out the default. `None` keeps the shard manager's own default. - pub shard_lease_duration_override: Option, pub oplog_archive_interval: Option, pub shared_client: bool, pub db_type: DbType, @@ -240,7 +237,6 @@ impl Default for EnvBasedTestDependenciesConfig { worker_executor_cluster_size: 4, environment_state_cache_capacity: None, number_of_shards_override: None, - shard_lease_duration_override: None, oplog_archive_interval: None, shared_client: false, db_type: DbType::Postgres, @@ -349,7 +345,6 @@ impl EnvBasedTestDependencies { &config.debug_targets_dirs().join("golem-shard-manager"), &config.golem_repo_root.join("golem-shard-manager"), config.number_of_shards_override, - config.shard_lease_duration_override, 9021, 9020, rdb, diff --git a/golem-worker-executor-test-utils/src/lib.rs b/golem-worker-executor-test-utils/src/lib.rs index 1dffe56b87..f4065d12d0 100644 --- a/golem-worker-executor-test-utils/src/lib.rs +++ b/golem-worker-executor-test-utils/src/lib.rs @@ -121,12 +121,11 @@ use golem_worker_executor::services::environment_state::EnvironmentStateService; use golem_worker_executor::services::file_loader::FileLoader; use golem_worker_executor::services::golem_config::{ AgentTypesServiceConfig, AgentTypesServiceLocalConfig, EngineConfig, - EnvironmentStateServiceConfig, FilesystemObjectLimitPolicyConfig, FilesystemPressureConfig, - GolemConfig, GrpcApiConfig, HttpClientConfig, IndexedStorageConfig, - IndexedStorageKVStoreRedisConfig, IndexedStorageKVStoreSqliteConfig, KeyValueStorageConfig, - KeyValueStorageInnerConfig, KeyValueStorageNamespaceRoutedConfig, MemoryConfig, OplogConfig, - ResourceLimitsConfig, ResourceLimitsDisabledConfig, ResourceUsageMeteringConfig, - SchedulerStorageConfig, SnapshotPolicy, + EnvironmentStateServiceConfig, GolemConfig, GrpcApiConfig, HttpClientConfig, + IndexedStorageConfig, IndexedStorageKVStoreRedisConfig, IndexedStorageKVStoreSqliteConfig, + KeyValueStorageConfig, KeyValueStorageInnerConfig, KeyValueStorageNamespaceRoutedConfig, + MemoryConfig, OplogConfig, ResourceLimitsConfig, ResourceLimitsDisabledConfig, + ResourceUsageMeteringConfig, SchedulerStorageConfig, SnapshotPolicy, }; use golem_worker_executor::services::key_value::{DefaultKeyValueService, KeyValueService}; use golem_worker_executor::services::oplog::{ @@ -154,6 +153,8 @@ use golem_worker_executor::services::worker_event::WorkerEventService; use golem_worker_executor::services::worker_fork::WorkerForkService; use golem_worker_executor::services::worker_proxy::{RemoteWorkerProxy, WorkerProxy}; use golem_worker_executor::services::{HasAll, NoAdditionalDeps, rdbms}; +use golem_worker_executor::storage::indexed::sqlite::SqliteIndexedStorage; +use golem_worker_executor::storage::indexed::{IndexedStorage, IndexedStorageNamespace}; use golem_worker_executor::storage::keyvalue::KeyValueStorage; use golem_worker_executor::worker::{RetryDecision, Worker}; use golem_worker_executor::workerctx::{ @@ -162,7 +163,9 @@ use golem_worker_executor::workerctx::{ InvocationManagement, LogEventEmitBehaviour, P3HttpBodyProducerHook, StatusManagement, UpdateManagement, WorkerCtx, WorkerFilesystemContext, }; -use golem_worker_executor::{Bootstrap, RunDetails, bootstrap_and_run_worker_executor}; +use golem_worker_executor::{ + Bootstrap, RunDetails, bootstrap_and_run_worker_executor, derive_disjoint_sqlite_config, +}; use prometheus::Registry; use regex::Regex; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -1603,6 +1606,41 @@ pub fn scheduler_sqlite_storage_config( } } +/// Raises the owning epoch stored for `owned_agent_id`'s oplog to `epoch`, as a newer owner +/// opening it on another executor would. The executor's own shard assignment is left alone, so +/// its next oplog write is refused: this is the zombie side of a shard move. +/// +/// Reaches the storage of executors started with the SQLite storage config (`start`, +/// `start_with_overrides`, `start_customized`), whose indexed storage lives in its own file next to +/// the key-value one. +pub async fn take_agent_oplog_over_at_epoch( + deps: &WorkerExecutorTestDependencies, + context: &TestContext, + owned_agent_id: &OwnedAgentId, + epoch: u64, +) -> anyhow::Result<()> { + let storage = SqliteIndexedStorage::configured(&derive_disjoint_sqlite_config( + &sqlite_storage_config(deps, context), + "indexed", + )) + .await + .map_err(|err| anyhow!(err))?; + // The namespace and key the executor's own open records its epoch under. + storage + .upsert_oplog_metadata( + "oplog", + "test_take_over", + IndexedStorageNamespace::OpLog { + agent_id: owned_agent_id.agent_id(), + agent_mode: AgentMode::Durable, + }, + &owned_agent_id.agent_id.to_redis_key(), + ShardEpoch(epoch), + ) + .await?; + Ok(()) +} + fn apply_sqlite_storage_config( config: &mut GolemConfig, deps: &WorkerExecutorTestDependencies, @@ -3544,7 +3582,10 @@ impl TestOplog { .has_fire_and_forget_rpc_commit_gate(&self.owned_agent_id.agent_id, checkpoint) .await { - self.oplog.commit(CommitLevel::Always).await; + self.oplog + .commit(CommitLevel::Always) + .await + .expect("oplog commit failed at the fire-and-forget RPC gate"); self.additional_test_deps .pause_after_fire_and_forget_rpc_commit(&self.owned_agent_id.agent_id, checkpoint) .await; @@ -5283,6 +5324,23 @@ impl ShardService for FakeOwnership { fn try_get_current_assignment(&self) -> Option { self.inner.try_get_current_assignment() } + + fn fence_learned_epochs(&self) -> std::collections::BTreeMap { + self.inner.fence_learned_epochs() + } + + fn retire_fence_learned_epochs( + &self, + reported: &std::collections::BTreeMap, + ) { + self.inner.retire_fence_learned_epochs(reported) + } +} + +impl golem_worker_executor::services::oplog::OplogFenceObserver for FakeOwnership { + fn fenced(&self, fence: &golem_worker_executor::services::oplog::OplogFence) { + self.inner.fenced(fence) + } } /// A test's handle on a [`FakeOwnership`]: what it should report, when to hold diff --git a/golem-worker-executor/src/durable_host/concurrent/call.rs b/golem-worker-executor/src/durable_host/concurrent/call.rs index 62c67981f6..bd76b8aa0e 100644 --- a/golem-worker-executor/src/durable_host/concurrent/call.rs +++ b/golem-worker-executor/src/durable_host/concurrent/call.rs @@ -2107,8 +2107,16 @@ impl DurableCallSession { } ScopeReplayRecovery::Default => {} } - let begin_index = - Self::append_access_scope_start(prepared, scope_name, function_type).await; + let begin_index = Self::append_access_scope_start(prepared, scope_name, function_type) + .await + .map_err(|error| { + ( + error, + AccessStartCleanup { + atomic_lease: prepared.atomic_lease.clone(), + }, + ) + })?; Ok(AccessOpenedScope { begin_index, replay_handle: None, @@ -2304,7 +2312,16 @@ impl DurableCallSession { }; let Some((begin_index, replay_handle)) = claimed_scope else { let begin_index = - Self::append_access_scope_start(prepared, scope_name, function_type).await; + Self::append_access_scope_start(prepared, scope_name, function_type) + .await + .map_err(|error| { + ( + error, + AccessStartCleanup { + atomic_lease: prepared.atomic_lease.clone(), + }, + ) + })?; prepared .public_state .worker() @@ -2470,15 +2487,18 @@ impl DurableCallSession { } } + /// Appends the scope `Start` that the call's side effect waits on. A `Start` the storage + /// refused is returned as the fence, so the effect never runs for a scope the shard's new + /// owner cannot see. async fn append_access_scope_start( prepared: &mut PreparedAccessStart, scope_name: HostFunctionName, function_type: DurableFunctionType, - ) -> OplogIndex { + ) -> Result { prepared .public_state .worker() - .add_and_commit_oplog(OplogEntry::Start { + .add_and_commit_oplog_or_fenced(OplogEntry::Start { timestamp: Timestamp::now_utc(), parent_start_index: prepared.entity_parent_start_index, function_name: scope_name, @@ -2488,6 +2508,7 @@ impl DurableCallSession { durable_function_type: function_type, }) .await + .map_err(WorkerExecutorError::from) } fn finish_access_start( diff --git a/golem-worker-executor/src/durable_host/durable_stream.rs b/golem-worker-executor/src/durable_host/durable_stream.rs index 8dea7d8c8d..e9461e72f0 100644 --- a/golem-worker-executor/src/durable_host/durable_stream.rs +++ b/golem-worker-executor/src/durable_host/durable_stream.rs @@ -21,7 +21,8 @@ use crate::durable_host::stream_bus::{ #[cfg(test)] use crate::services::oplog::CommitLevel; use crate::services::oplog::{ - DurableStreamOplogRecord, Oplog, OplogOps, OplogService, OplogServiceOps, + DurableStreamOplogRecord, Oplog, OplogError, OplogFence, OplogOps, OplogService, + OplogServiceOps, }; use crate::services::rpc::Rpc; use crate::services::worker::WorkerService; @@ -54,6 +55,7 @@ use golem_common::model::oplog::payload::OplogPayload; use golem_schema::schema::{ SchemaFingerprintV1, SchemaGraph, SchemaType, SchemaValue, TypedSchemaValue, }; +use golem_service_base::error::worker_executor::WorkerExecutorError; use golem_service_base::model::auth::AuthCtx; use metadata::{ProducerMetadataKey, ProducerMetadataRow}; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -171,10 +173,23 @@ pub(crate) enum DurableStreamProducerError { ConsumerJournalAdvanced, DeletionBlocked(Vec), CorruptHistory(String), + Fenced(OplogFence), Oplog(String), LiveBus(DurableLiveStreamBusError), } +/// A refused write keeps its type as `Fenced` instead of joining `Oplog` as text, so the +/// boundaries can report it as `OplogFenced` - a caller reroutes on that - rather than as a +/// failure of the request. +impl From for DurableStreamProducerError { + fn from(error: OplogError) -> Self { + match error { + OplogError::Fenced(fence) => Self::Fenced(fence), + error @ OplogError::Storage(_) => Self::Oplog(error.to_string()), + } + } +} + impl std::fmt::Display for DurableStreamProducerError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(formatter, "{self:?}") @@ -184,6 +199,18 @@ impl std::fmt::Display for DurableStreamProducerError { impl std::error::Error for DurableStreamProducerError {} impl DurableStreamProducerError { + /// Converts at a boundary that reports `WorkerExecutorError`: a fence keeps its type, and every + /// other error is rendered through `otherwise`, which says how that boundary classifies it. + pub(crate) fn into_worker_executor_error( + self, + otherwise: impl FnOnce(String) -> WorkerExecutorError, + ) -> WorkerExecutorError { + match self { + Self::Fenced(fence) => WorkerExecutorError::from(OplogError::Fenced(fence)), + error => otherwise(error.to_string()), + } + } + pub(crate) fn deletion_blocked_evidence(&self) -> Option { let Self::DeletionBlocked(dependents) = self else { return None; @@ -2091,12 +2118,28 @@ impl DurableStreamProducer { })) } - async fn commit(&self) { + async fn commit(&self) -> Result<(), DurableStreamProducerError> { (self.commit)(None).await; + self.committed_unless_fenced() } - async fn commit_notifying(&self, committed: oneshot::Sender<()>) { + async fn commit_notifying( + &self, + committed: oneshot::Sender<()>, + ) -> Result<(), DurableStreamProducerError> { (self.commit)(Some(committed)).await; + self.committed_unless_fenced() + } + + /// The worker's commit swallows a refusal: it only spawns the relinquish. The refused append + /// has latched the fence before the commit resolves, so the latch is what tells a persisted + /// write from one that must not be indexed, retained or published. A below-threshold add + /// answers `Ok` on a latched oplog, so no earlier result can stand in for this check. + fn committed_unless_fenced(&self) -> Result<(), DurableStreamProducerError> { + match self.oplog.fence() { + Some(fence) => Err(DurableStreamProducerError::Fenced(fence)), + None => Ok(()), + } } fn retain_committed_events(&self, events: &[CommittedProducerStreamEventV1]) { @@ -2401,11 +2444,11 @@ impl DurableStreamProducer { OplogPayload::Inline(Box::new(record)), )) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; if let Some(key) = result_key { index.invocation_results.entry(key).or_insert(oplog_index); } - self.commit().await; drop(index); self.notify_session_records_changed(); Ok(()) @@ -2532,8 +2575,8 @@ impl DurableStreamProducer { OplogPayload::Inline(Box::new(record)), )) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; self.notify_session_records_changed(); Ok(false) } @@ -2599,8 +2642,8 @@ impl DurableStreamProducer { OplogPayload::Inline(Box::new(record)), )) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; *index = updated; } drop(index); @@ -2759,8 +2802,8 @@ impl DurableStreamProducer { )] })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let (oplog_index, entry) = entries .pop() .expect("registration batch returned no oplog entry"); @@ -2912,7 +2955,7 @@ impl DurableStreamProducer { result })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; + .map_err(DurableStreamProducerError::from)?; let mut prepared = None; let mut registrations = Vec::with_capacity(requests.len()); @@ -2979,7 +3022,7 @@ impl DurableStreamProducer { &StreamSessionRecordV1::Prepared(prepared.clone()), )?; - self.commit_notifying(committed).await; + self.commit_notifying(committed).await?; *index = updated_index; self.buses .write() @@ -3110,8 +3153,8 @@ impl DurableStreamProducer { )] })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let (oplog_index, entry) = entries.pop().ok_or_else(|| { DurableStreamProducerError::CorruptHistory( "empty result registration batch returned no session record".to_string(), @@ -3240,8 +3283,8 @@ impl DurableStreamProducer { result })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let mut handles = Vec::new(); let mut session_record = None; @@ -3937,8 +3980,8 @@ impl DurableStreamProducer { records })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let mut pending_registrations = Vec::new(); let mut committed_item = None; @@ -4048,8 +4091,8 @@ impl DurableStreamProducer { )] })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let (oplog_index, entry) = entries .pop() .expect("resource exhaustion terminal batch returned no oplog entry"); @@ -4187,8 +4230,8 @@ impl DurableStreamProducer { )] })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let (oplog_index, entry) = entries .pop() .expect("stream end batch returned no oplog entry"); @@ -4321,8 +4364,8 @@ impl DurableStreamProducer { )] })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let (oplog_index, entry) = entries .pop() .expect("stream cancellation batch returned no oplog entry"); @@ -4655,8 +4698,8 @@ impl DurableStreamProducer { records })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let mut terminal_events = Vec::new(); for (oplog_index, entry) in entries { @@ -6112,8 +6155,8 @@ impl DurableStreamProducer { records })) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; let mut terminal_events = Vec::new(); for (oplog_index, entry) in entries { match entry { @@ -6219,8 +6262,8 @@ impl DurableStreamProducer { OplogPayload::Inline(Box::new(record.clone())), )) .await - .map_err(|error| DurableStreamProducerError::Oplog(error.to_string()))?; - self.commit().await; + .map_err(DurableStreamProducerError::from)?; + self.commit().await?; index.apply_session_references(entity_parent_start_index, &record)?; index.apply_deletion_record( &record, @@ -6963,6 +7006,11 @@ pub(crate) mod tests { entries: BTreeMap, committed: OplogIndex, commit_count: u64, + /// When set, `add` answers the way a primary oplog does when the threshold commit behind + /// the add is refused by the storage. + refused_adds: Option, + /// The fence a primary oplog latches when the storage refuses one of its commits. + fence: Option, } #[derive(Default)] @@ -7000,6 +7048,14 @@ pub(crate) mod tests { .cloned() .collect() } + + fn refuse_adds(&self, fence: crate::services::oplog::OplogFence) { + self.state.lock().unwrap().refused_adds = Some(fence); + } + + fn latch_fence(&self, fence: crate::services::oplog::OplogFence) { + self.state.lock().unwrap().fence = Some(fence); + } } #[async_trait] @@ -7009,6 +7065,9 @@ pub(crate) mod tests { entry: OplogEntry, ) -> Result { let mut state = self.state.lock().unwrap(); + if let Some(fence) = &state.refused_adds { + return Err(crate::services::oplog::OplogError::Fenced(fence.clone())); + } let index = state .entries .last_key_value() @@ -7027,6 +7086,10 @@ pub(crate) mod tests { Box::pin(async move { Ok(index) }) } + fn fence(&self) -> Option { + self.state.lock().unwrap().fence.clone() + } + async fn drop_prefix(&self, last_dropped_id: OplogIndex) -> u64 { let mut state = self.state.lock().unwrap(); let before = state.entries.len(); @@ -10328,6 +10391,164 @@ pub(crate) mod tests { ); } + fn test_fence() -> crate::services::oplog::OplogFence { + crate::services::oplog::OplogFence { + agent_id: identity().agent_id, + expected_epoch: golem_common::model::ShardEpoch(3), + actual_epoch: Some(golem_common::model::ShardEpoch(4)), + } + } + + /// A fence has to reach the worker executor's boundaries as a fence: flattened into + /// `InvalidRequest` it is rejected as a validation failure, and the caller never reroutes. + #[test] + fn a_fenced_oplog_error_keeps_its_type_through_the_producer() { + use golem_service_base::error::worker_executor::WorkerExecutorError; + + let fenced = DurableStreamProducerError::from(crate::services::oplog::OplogError::Fenced( + test_fence(), + )); + assert!(matches!(fenced, DurableStreamProducerError::Fenced(_))); + assert!(matches!( + fenced.into_worker_executor_error(WorkerExecutorError::invalid_request), + WorkerExecutorError::OplogFenced { .. } + )); + + let storage = DurableStreamProducerError::from( + crate::services::oplog::OplogError::Storage("connection reset".to_string()), + ); + assert!(matches!(storage, DurableStreamProducerError::Oplog(_))); + assert!(matches!( + storage.into_worker_executor_error(WorkerExecutorError::invalid_request), + WorkerExecutorError::InvalidRequest { .. } + )); + } + + #[test] + async fn a_fenced_session_record_append_is_reported_as_fenced() { + let identity = identity(); + let oplog = Arc::new(TestOplog::default()); + let producer = producer(oplog.clone(), &identity, None).await; + oplog.refuse_adds(test_fence()); + + let result = producer + .append_session_record(StreamSessionRecordV1::ConsumerDeleting( + StreamConsumerDeletingRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + consumer_environment_id: identity.environment_id, + consumer: identity.agent_id, + consumer_fingerprint: identity.fingerprint, + deleting_at_millis: 100, + }, + )) + .await; + + assert!( + matches!(result, Err(DurableStreamProducerError::Fenced(_))), + "a refused session record append must stay a fence, got {result:?}" + ); + } + + /// A producer committing the way the worker does: a commit refused by a fence is swallowed, + /// and only the oplog's latch records it. + async fn producer_swallowing_fenced_commits( + oplog: Arc, + identity: &TestIdentity, + ) -> Arc { + let commit_oplog = oplog.clone(); + let commit: DurableStreamCommit = Arc::new(move |committed| { + let oplog = commit_oplog.clone(); + Box::pin(async move { + if oplog.fence().is_some() { + return; + } + oplog + .commit(CommitLevel::Always) + .await + .expect("oplog write"); + if let Some(committed) = committed { + let _ = committed.send(()); + } + }) + }); + DurableStreamProducer::load_with_commit( + oplog, + identity.environment_id, + identity.agent_id.clone(), + identity.fingerprint, + None, + commit, + ) + .await + .unwrap() + } + + /// Items whose commit was refused never reached the oplog: a live reader handed them would + /// journal offsets the shard's new owner replays without. + #[test] + async fn a_fenced_commit_neither_publishes_nor_indexes_stream_items() { + let identity = identity(); + let oplog = Arc::new(TestOplog::default()); + let producer = producer_swallowing_fenced_commits(oplog.clone(), &identity).await; + let handle = producer + .register(root_registration(&identity)) + .await + .unwrap() + .value; + let bus = producer.bus(handle.stream_id).unwrap(); + let mut subscription = bus.subscribe().await.unwrap(); + let committed_before = oplog.committed_length(); + oplog.latch_fence(test_fence()); + + let result = producer + .write_items(handle.stream_id, 0, StreamItemsPayloadV1::PackedU8(vec![7])) + .await; + + assert!( + matches!(result, Err(DurableStreamProducerError::Fenced(_))), + "a write whose commit was refused must report the fence, got {result:?}" + ); + assert!( + tokio::time::timeout(Duration::from_millis(200), subscription.recv()) + .await + .is_err(), + "nothing may be published for a write whose commit was refused" + ); + assert_eq!(oplog.committed_length(), committed_before); + assert_eq!( + producer.index.lock().await.streams[&handle.stream_id].next_sequence, + 0 + ); + } + + #[test] + async fn a_fenced_commit_does_not_record_an_attachment_in_the_index() { + let identity = identity(); + let oplog = Arc::new(TestOplog::default()); + let producer = producer_swallowing_fenced_commits(oplog.clone(), &identity).await; + let handle = producer + .register(root_registration(&identity)) + .await + .unwrap() + .value; + let key = attachment_key(&identity, handle.stream_id); + oplog.latch_fence(test_fence()); + + let result = producer.prepare_attachment(key.clone(), 100).await; + + assert!( + matches!(result, Err(DurableStreamProducerError::Fenced(_))), + "an attachment whose commit was refused must report the fence, got {result:?}" + ); + assert!( + producer + .inspect_attachments() + .await + .iter() + .all(|view| view.key != key) + ); + } + #[test] async fn malformed_session_record_is_rejected_at_the_write_boundary() { let identity = identity(); diff --git a/golem-worker-executor/src/durable_host/golem/v1x.rs b/golem-worker-executor/src/durable_host/golem/v1x.rs index f9220639f0..cefdd3af9e 100644 --- a/golem-worker-executor/src/durable_host/golem/v1x.rs +++ b/golem-worker-executor/src/durable_host/golem/v1x.rs @@ -36,7 +36,7 @@ use crate::preview2::golem_api_1_x::host::{ use crate::preview2::golem_api_1_x::oplog::{ Host as OplogHost, HostGetOplog, HostSearchOplog, OplogReadError, SearchOplog, }; -use crate::services::oplog::CommitLevel; +use crate::services::oplog::{CommitLevel, OplogError}; use crate::services::promise::{PromiseHandle, PromiseService}; use crate::services::worker_proxy::WorkerProxyError; use crate::services::{HasOplogService, HasWorker}; @@ -727,7 +727,17 @@ impl Host for DurableWorkerCtx { debug!("Worker committing oplog to {replicas} replicas"); loop { // Applying a timeout to make sure the worker remains interruptible - if self.state.oplog.wait_for_replicas(replicas, timeout).await { + let committed = self.state.oplog.wait_for_replicas(replicas, timeout).await; + // The shard has a new owner, so nothing was committed and nothing can be. The + // fence surfaces as `ShardLost`, which gives the agent up without writing, instead + // of acknowledging a commit that did not happen or retrying one that never will: + // `check_interrupt` below has no interrupt to report for a latched fence. + if let Some(fence) = self.state.oplog.fence() { + return Err(anyhow!(WorkerExecutorError::from(OplogError::Fenced( + fence + )))); + } + if committed { debug!("Worker committed oplog to {replicas} replicas"); return Ok(()); } else { @@ -891,7 +901,8 @@ impl Host for DurableWorkerCtx { self.entity_parent_start_index(), begin_index, )) - .await?; + .await + .map_err(|error| anyhow!(WorkerExecutorError::from(error)))?; } else { let (_, _) = get_oplog_entry!(self.state.replay_state, OplogEntry::EndAtomicRegion)?; } diff --git a/golem-worker-executor/src/durable_host/mod.rs b/golem-worker-executor/src/durable_host/mod.rs index e2d7d7e901..1a55c71582 100644 --- a/golem-worker-executor/src/durable_host/mod.rs +++ b/golem-worker-executor/src/durable_host/mod.rs @@ -628,6 +628,12 @@ fn validate_unshared_memory_growth( } impl DurableWorkerCtx { + /// `trap_type`, or `ShardLost` once this agent's oplog has latched a fence. For the invocation + /// loop, which reaches the oplog only through this context. + pub(crate) fn trap_type_under_latched_fence(&self, trap_type: TrapType) -> TrapType { + trap_type.under_latched_fence(self.state.oplog.fence().as_ref()) + } + #[cfg(feature = "test-utils")] pub(crate) fn test_should_skip_wall_clock_now_durability(&self) -> bool { self.owner_execution @@ -2777,7 +2783,15 @@ impl DurableWorkerCtx { request: None, durable_function_type: function_type.clone(), }; - let begin_index = self.public_state.worker().add_and_commit_oplog(entry).await; + // The scope's side effect runs as soon as this returns. A `Start` the storage + // refused has to stop it here: the shard's new owner has no record of the scope, + // so nothing would stop it running the effect a second time. + let begin_index = self + .public_state + .worker() + .add_and_commit_oplog_or_fenced(entry) + .await + .map_err(WorkerExecutorError::from)?; Ok(begin_index) } else { let scope_name = HostFunctionName::Custom("".to_string()); @@ -3101,10 +3115,14 @@ impl DurableWorkerCtx { ) .await .map_err(WorkerExecutorError::from)?; + // The pair is only buffered until this commit. If the storage refused it, the + // transaction must not be handed out, so `tx` is dropped here before any statement + // has run through it. self.public_state .worker() - .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .commit_oplog_or_fenced(CommitLevel::Always) + .await + .map_err(WorkerExecutorError::from)?; // The transaction scope is now open until commit/rollback; block checkpoints. Opened // live, so there is no recorded scope `End` to await on close. @@ -3266,13 +3284,16 @@ impl DurableWorkerCtx { end: pending.replay_target().next(), // skipping the Jump entry too }; + // Checked, like the begin below: a refused jump means the restart is not ours + // to run, and stopping here avoids opening a database transaction first. self.public_state .worker() - .add_and_commit_oplog(OplogEntry::jump( + .add_and_commit_oplog_or_fenced(OplogEntry::jump( self.entity_parent_start_index(), deleted_region, )) - .await; + .await + .map_err(WorkerExecutorError::from)?; // TODO: this recomputation should not be necessary. self.public_state.worker().reattach_worker_status().await; @@ -3280,14 +3301,16 @@ impl DurableWorkerCtx { self.finish_switch_to_live(pending).await?.require_live()?; let (tx_id, tx) = handler.create_new().await?; - let _ = self - .public_state + // The restarted transaction runs its statements once this returns, so a + // refused begin has to stop it; `tx` is dropped unused. + self.public_state .worker() - .add_and_commit_oplog(OplogEntry::begin_remote_transaction( + .add_and_commit_oplog_or_fenced(OplogEntry::begin_remote_transaction( tx_id, Some(original_begin_index), )) - .await; + .await + .map_err(WorkerExecutorError::from)?; // Restarted live (jump + fresh `BeginRemoteTransaction`): the scope `End` will // be appended live by the transaction terminal, so do not store the (now @@ -4806,6 +4829,9 @@ impl InvocationHooks for DurableWorkerCtx { full_function_name: &str, trap_type: &TrapType, ) -> RetryDecision { + // Covers the callers that hand over a trap and do not branch on it afterwards; the ones + // that do reclassify before this call, so they dispatch on the same kind. + let trap_type = &self.trap_type_under_latched_fence(trap_type.clone()); let current_idempotency_key = self.get_current_idempotency_key().await; // Deliberately above the dropped-call drain: that drain appends `Cancelled` entries, and @@ -5802,7 +5828,18 @@ impl ExternalOperations for DurableWorkerCtx { store.as_context().data().agent_mode(), )) } - }; + } + // Every arm below dispatches on the kind. Left an `Error`, a fence + // whose type was lost on the way would abandon the snapshot or break + // with `InvocationFailed` into a recovery failure, which unloads the + // agent as failed instead of relinquishing it. + .map(|trap_type| { + store + .as_context() + .data() + .durable_ctx() + .trap_type_under_latched_fence(trap_type) + }); let decision = match trap_type { // A recorded invocation that fails while its entries are still // being replayed after an automatic snapshot load most likely @@ -6166,22 +6203,20 @@ impl ExternalOperations for DurableWorkerCtx { // TODO: there is probably a race here between assignment changing and a suspended worker getting woken up. if should_restart_after_shard_assignment_change(&latest_worker_status) { - Worker::get_or_create_running( - this, + recovered_restart( &owned_agent_id, - None, - Vec::new(), - None, - None, - &InvocationContextStack::fresh(), - Principal::anonymous(), - ) - .await - .map_err(|error| { - anyhow!( - "failed to restart {owned_agent_id} during shard-assignment recovery: {error}" + Worker::get_or_create_running( + this, + &owned_agent_id, + None, + Vec::new(), + None, + None, + &InvocationContextStack::fresh(), + Principal::anonymous(), ) - })?; + .await, + )?; } } @@ -6594,6 +6629,29 @@ fn recovered_status( } } +/// The outcome of restarting one recovered agent, drawing the same line as [`recovered_status`]. +/// +/// `ShardingNotReady` means the agent's shard left this executor's assignment while recovery was +/// running: a later delivery revoked it, and the worker was refused an epoch rather than opened +/// unfenced. The agent belongs to the shard's new owner, which recovers it there. This is a skip, +/// like an oplog that is gone. Failing the whole assignment for it would stop every other agent in +/// the scan from being resumed. Any other restart failure still fails the assignment. +fn recovered_restart( + owned_agent_id: &OwnedAgentId, + restarted: Result, +) -> Result<(), anyhow::Error> { + match restarted { + Ok(_) => Ok(()), + Err(WorkerExecutorError::ShardingNotReady) => { + debug!(agent_id = %owned_agent_id, "Worker's shard left the assignment during shard-assignment recovery; skipping agent"); + Ok(()) + } + Err(error) => Err(anyhow!( + "failed to restart {owned_agent_id} during shard-assignment recovery: {error}" + )), + } +} + fn should_restart_after_shard_assignment_change(status: &AgentStatusRecord) -> bool { matches!( status.status, @@ -8536,6 +8594,31 @@ mod tests { ); } + /// A revoke that races recovery refuses the worker an epoch. That agent is skipped, so the rest + /// of the scan is still resumed. Any other restart failure still fails the assignment. + #[test] + fn shard_assignment_recovery_skips_a_worker_whose_shard_left_the_assignment() { + assert!(recovered_restart(&recovered_agent(), Ok(())).is_ok()); + assert!( + recovered_restart::<()>( + &recovered_agent(), + Err(WorkerExecutorError::ShardingNotReady) + ) + .is_ok() + ); + + let error = recovered_restart::<()>( + &recovered_agent(), + Err(WorkerExecutorError::runtime("instance failed to start")), + ) + .expect_err("a restart failure other than a lost shard was skipped"); + assert!( + error.to_string().contains("failed to restart") + && error.to_string().contains("instance failed to start"), + "{error}" + ); + } + fn open_region(regions: &mut Vec, begin: u64) -> OplogIndex { let begin_index = OplogIndex::from_u64(begin); regions.push(ActiveAtomicRegion::new(begin_index, begin_index.next())); diff --git a/golem-worker-executor/src/durable_host/p3/http/replay.rs b/golem-worker-executor/src/durable_host/p3/http/replay.rs index 23207b9d3d..bdd5758e93 100644 --- a/golem-worker-executor/src/durable_host/p3/http/replay.rs +++ b/golem-worker-executor/src/durable_host/p3/http/replay.rs @@ -37,7 +37,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::task::Poll; use tokio::sync::{Notify, oneshot}; -use tracing::warn; +use tracing::{debug, warn}; use wasmtime::AsContextMut; use wasmtime::component::{Accessor, AccessorTask, Resource}; use wasmtime_wasi_http::p3::WasiHttp; @@ -556,6 +556,16 @@ async fn record_frame_or_warn( ) -> bool { match record_frame_entry(oplog.clone(), send_start_index, frame).await { Ok(_) => true, + // Not a recording fault: the storage turned the write away because the shard has a new + // owner, and the latch gives the agent up on its next durable write. + Err(error) if oplog.fence().is_some() => { + debug!( + send_start_index = %send_start_index, + error = %error, + "Request-body frame not recorded: the shard moved; the next durable write gives the agent up" + ); + false + } Err(error) => { warn!( send_start_index = %send_start_index, diff --git a/golem-worker-executor/src/durable_host/p3/http/send.rs b/golem-worker-executor/src/durable_host/p3/http/send.rs index 64872fb3b2..83d838fab2 100644 --- a/golem-worker-executor/src/durable_host/p3/http/send.rs +++ b/golem-worker-executor/src/durable_host/p3/http/send.rs @@ -34,6 +34,7 @@ use crate::durable_host::http::policy::{ use crate::durable_host::http::types::classify_serializable_http_error_code; use crate::durable_host::p3::{DurableP3, DurableP3View, durable_worker_ctx, wasi_http_view}; use crate::services::HasWorker; +use crate::services::oplog::{Oplog, OplogError}; use crate::workerctx::WorkerCtx; use anyhow::Context as _; use bytes::Bytes; @@ -856,6 +857,27 @@ where } Err(error_code) => { let _ = physical.final_transmission_tx.send(Err(error_code.clone())); + + // A write the storage refused because the shard moved latches the oplog, and nothing + // on this path sees the latch otherwise: a below-threshold frame add still succeeds, + // and so does this send's buffered `End`. Without this read the guest would be + // handed an HTTP error produced by, or racing, the lost shard instead of the + // ShardLost trap. `handle.trap` abandons the call exactly as the retry trap below + // does. + let latched = store.with(|mut access| { + latched_fence_error( + durable_worker_ctx::(access.data_mut()) + .state + .oplog + .as_ref(), + ) + }); + if let Some(error) = latched { + return Err(HttpError::trap(wasmtime::Error::from_anyhow( + handle.trap(error), + ))); + } + let serialized_error = serialize_error_code(&error_code); // Worker-level retry classification, mirroring the P2 @@ -948,6 +970,13 @@ where } } +/// The error a send traps with once the oplog has latched a fence, or `None` while it has not. +fn latched_fence_error(oplog: &dyn Oplog) -> Option { + oplog + .fence() + .map(|fence| WorkerExecutorError::from(OplogError::Fenced(fence))) +} + pub(super) struct PhysicalSendHttpError { error_code: ErrorCode, final_transmission_tx: oneshot::Sender>, @@ -1487,3 +1516,75 @@ pub(super) fn apply_headers_to_request_resource( .map_err(WorkerExecutorError::runtime) }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::durable_host::durability::{DurableCallTrapContext, mark_durable_call_trap_context}; + use crate::durable_host::p3::http::test_support::*; + use crate::model::TrapType; + use crate::services::oplog::OplogFence; + use golem_common::model::agent::AgentMode; + use golem_common::model::component::ComponentId; + use golem_common::model::oplog::payload::types::SerializableP3HttpRequestBodyFrame; + use golem_common::model::{AgentId, ShardEpoch}; + use golem_service_base::error::worker_executor::InterruptKind; + use test_r::test; + + /// The send's gate cannot rely on the recording: once the fence has latched, a frame add that + /// stays below the commit threshold still succeeds. Only the latch says the shard moved, and + /// the error read from it has to keep its ShardLost classification through `handle.trap`. + /// + /// The gate inside `send_with_durability` needs a wasmtime `Accessor` and a live worker + /// context, so it is exercised here through the pieces it is built from. + #[test] + async fn a_send_failure_after_a_latched_fence_traps_as_shard_lost_even_when_frame_adds_succeed() + { + let oplog = FrameTestOplog::new(); + assert!( + latched_fence_error(oplog.as_ref()).is_none(), + "no fence has latched, so a send failure is a genuine HTTP error" + ); + + oplog.latch_fence(OplogFence { + agent_id: AgentId { + component_id: ComponentId::new(), + agent_id: "sender".to_string(), + }, + expected_epoch: ShardEpoch(8), + actual_epoch: Some(ShardEpoch(9)), + }); + record_frame_entry( + oplog.clone(), + OplogIndex::NONE, + SerializableP3HttpRequestBodyFrame::End, + ) + .await + .expect("a frame add below the commit threshold does not see the latch"); + + let error = latched_fence_error(oplog.as_ref()).expect("the latch must be read"); + assert!( + matches!(error, WorkerExecutorError::OplogFenced { .. }), + "expected a fenced error, got {error:?}" + ); + + let trapped = mark_durable_call_trap_context( + anyhow::Error::from(error), + DurableCallTrapContext { + retry_from: OplogIndex::INITIAL, + in_atomic_region: false, + }, + ); + let trap = TrapType::from_error::( + &trapped, + OplogIndex::INITIAL, + false, + false, + AgentMode::Durable, + ); + assert!( + matches!(trap, TrapType::Interrupt(InterruptKind::ShardLost)), + "the trapped send must give the agent up, got {trap:?}" + ); + } +} diff --git a/golem-worker-executor/src/durable_host/p3/http/test_support.rs b/golem-worker-executor/src/durable_host/p3/http/test_support.rs index 499d02515b..5e9ca0669c 100644 --- a/golem-worker-executor/src/durable_host/p3/http/test_support.rs +++ b/golem-worker-executor/src/durable_host/p3/http/test_support.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use crate::services::oplog::{CommitLevel, Oplog, OplogAddReceipt, OrderedOplogStart}; +use crate::services::oplog::{CommitLevel, Oplog, OplogAddReceipt, OplogFence, OrderedOplogStart}; use async_trait::async_trait; use bytes::Bytes; use golem_common::model::oplog::payload::types::{ @@ -43,6 +43,9 @@ use wasmtime_wasi_http::{FieldMap, WasiHttpCtx}; pub(super) struct FrameTestOplog { entries: std::sync::Mutex>, upload_gate: tokio::sync::Semaphore, + /// What `fence()` answers. `add` and `enqueue_add` keep succeeding while it is set, as the + /// primary oplog's below-threshold adds do after the fence has latched. + fence: std::sync::Mutex>, } impl FrameTestOplog { @@ -50,6 +53,7 @@ impl FrameTestOplog { Arc::new(Self { entries: std::sync::Mutex::new(Vec::new()), upload_gate: tokio::sync::Semaphore::new(tokio::sync::Semaphore::MAX_PERMITS), + fence: std::sync::Mutex::new(None), }) } @@ -59,9 +63,15 @@ impl FrameTestOplog { Arc::new(Self { entries: std::sync::Mutex::new(Vec::new()), upload_gate: tokio::sync::Semaphore::new(0), + fence: std::sync::Mutex::new(None), }) } + /// Latches `fence`, as a write the storage refused on another path would. + pub(super) fn latch_fence(&self, fence: OplogFence) { + *self.fence.lock().unwrap() = Some(fence); + } + pub(super) fn release_uploads(&self, n: usize) { self.upload_gate.add_permits(n); } @@ -210,6 +220,10 @@ impl Oplog for FrameTestOplog { self.entries.lock().unwrap().len() as u64 } + fn fence(&self) -> Option { + self.fence.lock().unwrap().clone() + } + async fn upload_raw_payload(&self, data: Vec) -> Result { let permit = self .upload_gate diff --git a/golem-worker-executor/src/durable_host/rdbms/mod.rs b/golem-worker-executor/src/durable_host/rdbms/mod.rs index 9e14a3eccb..acddd75896 100644 --- a/golem-worker-executor/src/durable_host/rdbms/mod.rs +++ b/golem-worker-executor/src/durable_host/rdbms/mod.rs @@ -21,6 +21,7 @@ use crate::durable_host::{ DurabilityHost, DurableWorkerCtx, InternalRetryResult, LiveAuthorizationPermit, RemoteTransactionHandler, }; +use crate::services::oplog::OplogError; use crate::services::rdbms::{DbResult, DbRow, RdbmsType}; use crate::services::rdbms::{RdbmsError, RdbmsService, RdbmsTransactionStatus, RdbmsTypeService}; use crate::workerctx::WorkerCtx; @@ -35,6 +36,7 @@ use golem_common::model::oplog::{ }; use golem_common::model::retry_policy::RetryProperties; use golem_common::model::{AgentId, OplogIndex, RdbmsPoolKey, RetryContext, TransactionId}; +use golem_service_base::error::worker_executor::WorkerExecutorError; use std::marker::PhantomData; use std::ops::Deref; use std::sync::Arc; @@ -301,7 +303,17 @@ where let resource = ctx.as_wasi_view().table().push(entry)?; Ok(Ok(resource)) } - Err(error) => Ok(Err(error.into())), + Err(error) => { + // The handler's error type flattens the begin's own fence into an `RdbmsError`, so it + // is read from the oplog's latch. A refused begin is a lost shard, not a database + // failure the guest may catch and work around: it traps, and the agent is given up. + if let Some(fence) = ctx.state.oplog.fence() { + return Err(anyhow!(WorkerExecutorError::from(OplogError::Fenced( + fence + )))); + } + Ok(Err(error.into())) + } } } diff --git a/golem-worker-executor/src/grpc/mod.rs b/golem-worker-executor/src/grpc/mod.rs index 76f5b2029e..56e15615db 100644 --- a/golem-worker-executor/src/grpc/mod.rs +++ b/golem-worker-executor/src/grpc/mod.rs @@ -31,14 +31,14 @@ use crate::services::worker_activator::{ }; use crate::services::worker_event::WorkerEventReceiver; use crate::services::{ - All, HasActiveAgents, HasAll, HasComponentService, HasEvents, HasOplogService, + All, HasActiveAgents, HasAll, HasComponentService, HasEvents, HasOplog, HasOplogService, HasPromiseService, HasRunningWorkerEnumerationService, HasShardManagerService, HasShardService, HasWorkerEnumerationService, HasWorkerService, UsesAllDeps, }; pub use crate::worker::{ PERMISSION_CARD_INSTALL_RECIPIENT_MISMATCH, PERMISSION_CARD_TRANSFER_PAYLOAD_CONFLICT, }; -use crate::worker::{RelinquishReason, Worker, WorkerUpdateMode}; +use crate::worker::{RelinquishReason, Worker, WorkerUpdateMode, relinquished_by_assignment}; use crate::workerctx::WorkerCtx; use futures::Stream; use futures::StreamExt; @@ -1095,7 +1095,8 @@ impl + UsesAllDeps + Send + Sync + /// The one receipt path for a delivered shard set, whichever way it came: /// a registration, an `AssignShards` push, or a renewal reply that - /// corrected the set. Sweeps the agents whose shard went away, then hands + /// corrected the set. Sweeps the agents whose shard went away or came back + /// at a higher epoch, then hands /// the executor the new set to recover agents for. The sweep runs for /// every path, because a renewal can narrow the set as well as widen it: /// a path without it would leave agents running on shards this executor @@ -1117,15 +1118,35 @@ impl + UsesAllDeps + Send + Sync + T: HasAll + Send + Sync + 'static, { let ticket = this.shard_manager_service().recovery_deferred(); - // Pure set membership on purpose: a lapsed lease must not give up every running agent - - // a lapsed lease refuses new work and leaves running work alone. + // Membership and epochs, never the lease: a lapsed lease must not give up every running + // agent - a lapsed lease refuses new work and leaves running work alone. // // Given up rather than restarted: a narrowing delivery means these shards have another - // owner now, and a restart in place would reopen their oplogs at the stale epoch. - let shard_service = this.shard_service(); + // owner now, and a restart in place would reopen their oplogs at the stale epoch. A + // delivery that raises the epoch of a shard this executor kept means the shard left and + // came back, so another executor may have written to its agents. Those are given up the + // same way, and the recovery below or their next invocation reopens them at the new epoch. + // + // The epochs come from one snapshot and the assignment from one read, both taken just + // before the sweep selects. An agent created after the snapshot read its epoch from the + // delivered assignment, so only membership applies to it. An agent given up and reopened + // at the new epoch between the snapshot and the selection is given up once more, which + // the same reopen repairs. + let held_epochs: HashMap> = this + .active_agents() + .snapshot() + .await + .into_iter() + .map(|(agent_id, worker)| (agent_id, worker.oplog().shard_epoch())) + .collect(); + let assignment = this.shard_service().try_get_current_assignment(); this.active_agents() .relinquish_matching(RelinquishReason::ShardNotAssigned, |agent_id| { - shard_service.check_worker(agent_id).is_err() + relinquished_by_assignment( + assignment.as_ref(), + agent_id, + held_epochs.get(agent_id).copied().flatten(), + ) }) .await; diff --git a/golem-worker-executor/src/lib.rs b/golem-worker-executor/src/lib.rs index 4641529848..efcf3d2d32 100644 --- a/golem-worker-executor/src/lib.rs +++ b/golem-worker-executor/src/lib.rs @@ -828,30 +828,24 @@ pub async fn create_worker_executor_impl< } let oplog_archives = NEVec::try_from_vec(oplog_archives); + // Built once for both shapes, so neither can be left without the observer: without it a + // refused write never reaches the shard manager, and a manager whose state lost history goes + // on minting below the oplog rows that refuse it. + let primary_oplog_service = PrimaryOplogService::new( + indexed_storage.clone(), + blob_storage.clone(), + golem_config.oplog.max_operations_before_commit, + golem_config.oplog.max_operations_before_commit_ephemeral, + golem_config.oplog.max_payload_size, + golem_config.indexed_storage_retry.clone(), + ) + .await + .with_fence_observer(shard_service.clone()); + let base_oplog_service: Arc = match oplog_archives { - None => Arc::new( - PrimaryOplogService::new( - indexed_storage.clone(), - blob_storage.clone(), - golem_config.oplog.max_operations_before_commit, - golem_config.oplog.max_operations_before_commit_ephemeral, - golem_config.oplog.max_payload_size, - golem_config.indexed_storage_retry.clone(), - ) - .await, - ), + None => Arc::new(primary_oplog_service), Some(oplog_archives) => { - let primary = Arc::new( - PrimaryOplogService::new( - indexed_storage.clone(), - blob_storage.clone(), - golem_config.oplog.max_operations_before_commit, - golem_config.oplog.max_operations_before_commit_ephemeral, - golem_config.oplog.max_payload_size, - golem_config.indexed_storage_retry.clone(), - ) - .await, - ); + let primary = Arc::new(primary_oplog_service); Arc::new(MultiLayerOplogService::new( primary, @@ -1063,7 +1057,11 @@ pub async fn create_worker_executor_impl< /// Derives a `DbSqliteConfig` for a module that should live in a separate /// SQLite DB file next to a base one (used by `KVStoreSqlite` to give the /// indexed storage its own DB and migration table). -fn derive_disjoint_sqlite_config(base: &DbSqliteConfig, suffix: &str) -> DbSqliteConfig { +/// +/// Public so test utilities can open the same file an executor uses without copying the naming +/// rule. +#[doc(hidden)] +pub fn derive_disjoint_sqlite_config(base: &DbSqliteConfig, suffix: &str) -> DbSqliteConfig { let database = match base.database.strip_suffix(".db") { Some(stem) => format!("{stem}-{suffix}.db"), None => format!("{}-{suffix}", base.database), diff --git a/golem-worker-executor/src/model/mod.rs b/golem-worker-executor/src/model/mod.rs index a2870e8c1e..b22690a269 100644 --- a/golem-worker-executor/src/model/mod.rs +++ b/golem-worker-executor/src/model/mod.rs @@ -292,6 +292,22 @@ pub enum TrapType { } impl TrapType { + /// `ShardLost` once the agent's oplog has latched a fence, whatever the trap was. + /// + /// A latched oplog refuses every later write, so giving the agent up is the only outcome + /// left. It also catches a fence that crossed a `String` boundary on its way to the trap and + /// no longer classifies as `ShardLost` by itself. + pub fn under_latched_fence( + self, + latched: Option<&crate::services::oplog::OplogFence>, + ) -> TrapType { + if latched.is_some() { + TrapType::Interrupt(InterruptKind::ShardLost) + } else { + self + } + } + pub fn from_worker_executor_error( error: WorkerExecutorError, fallback_retry_from: OplogIndex, @@ -515,8 +531,19 @@ impl TrapType { ))) } _ => { - // Search the full error chain for ClassifiedHostError - if let Some(classified) = error + // A bare `?` on an oplog write inside an anyhow host function + // carries the `OplogError` itself, not its `WorkerExecutorError` + // form, so the fence is looked for along the chain as well. A + // storage error stays a retriable `Unknown`. After that, search + // the full error chain for ClassifiedHostError. + if error.chain().any(|cause| { + matches!( + cause.downcast_ref::(), + Some(crate::services::oplog::OplogError::Fenced(_)) + ) + }) { + TrapType::Interrupt(InterruptKind::ShardLost) + } else if let Some(classified) = error .chain() .find_map(|e| e.downcast_ref::()) { @@ -973,6 +1000,105 @@ mod tests { ); } + fn fence_for(agent_id: &str) -> crate::services::oplog::OplogFence { + crate::services::oplog::OplogFence { + agent_id: golem_common::model::AgentId { + component_id: ComponentId::new(), + agent_id: agent_id.to_string(), + }, + expected_epoch: golem_common::model::ShardEpoch(7), + actual_epoch: Some(golem_common::model::ShardEpoch(8)), + } + } + + /// The same contract for a host function that puts a bare `?` on an oplog write: the error is + /// the `OplogError` itself, possibly under context, and still has to read as a lost shard. + #[test] + fn a_bare_fenced_oplog_error_escaping_a_host_call_classifies_as_shard_lost() { + let bare = anyhow::Error::from(crate::services::oplog::OplogError::Fenced(fence_for( + "bare-fenced-host-call", + ))); + let with_context = anyhow::Error::from(crate::services::oplog::OplogError::Fenced( + fence_for("bare-fenced-host-call"), + )) + .context("ending atomic region"); + + for error in [bare, with_context] { + let trap = TrapType::from_error::( + &error, + OplogIndex::INITIAL, + false, + false, + AgentMode::Durable, + ); + assert!( + matches!(trap, TrapType::Interrupt(InterruptKind::ShardLost)), + "a bare fenced oplog error must be an interrupt, got {trap:?}" + ); + } + } + + #[test] + fn a_bare_oplog_storage_error_is_not_shard_lost() { + let trap = TrapType::from_error::( + &anyhow::Error::from(crate::services::oplog::OplogError::Storage( + "connection reset".to_string(), + )), + OplogIndex::INITIAL, + false, + false, + AgentMode::Durable, + ); + + assert!( + !matches!(trap, TrapType::Interrupt(InterruptKind::ShardLost)), + "a bare storage failure must stay a retriable failure, got {trap:?}" + ); + } + + /// Once the oplog has latched a fence nothing more can be written for the agent, so no trap + /// may lead to a retry, an exit record or a jump - only to giving the agent up. + #[test] + fn a_latched_fence_turns_every_trap_into_shard_lost() { + let fence = fence_for("latched"); + let unknown_error = || TrapType::Error { + error: AgentError::Unknown("fence flattened into text".to_string()), + retry_from: OplogIndex::INITIAL, + in_atomic_region: false, + atomic_region_had_side_effects: false, + semantic_trap_retry_override: None, + }; + + for trap in [ + unknown_error(), + TrapType::Exit, + TrapType::Interrupt(InterruptKind::Jump), + TrapType::Interrupt(InterruptKind::Suspend(Timestamp::now_utc())), + ] { + let reclassified = trap.clone().under_latched_fence(Some(&fence)); + assert!( + matches!(reclassified, TrapType::Interrupt(InterruptKind::ShardLost)), + "{trap:?} under a latched fence must be a lost shard, got {reclassified:?}" + ); + let decision = crate::durable_host::DurableWorkerCtx::< + crate::workerctx::default::Context, + >::fixed_decision_for_trap_type(&reclassified); + assert_eq!(decision, Some(RetryDecision::None)); + } + + assert!(matches!( + unknown_error().under_latched_fence(None), + TrapType::Error { + error: AgentError::Unknown(_), + .. + } + )); + assert!(matches!( + TrapType::Interrupt(InterruptKind::Jump).under_latched_fence(None), + TrapType::Interrupt(InterruptKind::Jump) + )); + } + #[test] fn semantic_trap_retry_override_carries_retry_point() { use crate::durable_host::durability::{ diff --git a/golem-worker-executor/src/services/active_agents/mod.rs b/golem-worker-executor/src/services/active_agents/mod.rs index 925ebe7925..21281546c9 100644 --- a/golem-worker-executor/src/services/active_agents/mod.rs +++ b/golem-worker-executor/src/services/active_agents/mod.rs @@ -38,7 +38,7 @@ use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; -use tracing::{Instrument, debug}; +use tracing::{Instrument, debug, info}; use crate::durable_host::tool::operation::OwnerFailureWinner; use crate::services::HasAll; @@ -826,6 +826,63 @@ impl ActiveAgents { self.agents.remove(owned_agent_id).await } + /// The worker cached for `owned_agent_id`, without waiting on a creation still in progress. + /// + /// For callers acting on one particular generation: a pending entry is a newer generation + /// being created, never the one they hold, so waiting on it could only delay them. + pub(crate) async fn try_get_cached( + &self, + owned_agent_id: &OwnedAgentId, + ) -> Option>> { + self.agents + .try_get(owned_agent_id) + .await + .map(|active_agent| active_agent.primary()) + } + + /// Whether `worker` is the generation cached for its agent right now. + pub(crate) async fn is_cached_generation(&self, worker: &Worker) -> bool { + self.try_get_cached(worker.owned_agent_id()) + .await + .is_some_and(|cached| std::ptr::eq(Arc::as_ptr(&cached), worker)) + } + + /// [`Self::remove_with`] for one generation: tears the entry down and drops it only while it + /// still holds `worker`. Returns whether it did. + /// + /// A relinquished agent reaches its removal more than once - from its own loop's stop, again + /// from the relinquish that waited for it, or from a stop through a handle kept past its + /// generation - and by then a newer generation may be cached under the same id. Keyed by id + /// alone, such a pass evicts that generation and fences its entity bodies while its loop keeps + /// running. + /// + /// The final removal re-checks identity in the same map operation. The teardown before it does + /// not: should this generation be removed concurrently and a newer one cached in between, + /// clearing card interest reaches the newer one. That window is no wider than the one + /// [`Self::remove_with`] has. + pub(crate) async fn remove_generation( + &self, + worker: &Worker, + owner_failure: OwnerFailureWinner, + ) -> bool { + let owned_agent_id = worker.owned_agent_id(); + let is_this_generation = |active_agent: &Arc>| { + std::ptr::eq(Arc::as_ptr(&active_agent.primary), worker) + }; + match self.agents.try_get(owned_agent_id).await { + Some(active_agent) if is_this_generation(&active_agent) => { + active_agent.fence_entity_bodies(owner_failure).await; + self.card_interest_index + .set_card_interest(owned_agent_id.clone(), &[]) + .await; + } + _ => return false, + } + self.agents + .remove_if_cached(owned_agent_id, is_this_generation) + .await + } + pub async fn tracked_card_ids(&self) -> Vec { self.card_interest_index.tracked_card_ids().await } @@ -905,6 +962,14 @@ impl ActiveAgents { .map(|(_, worker)| worker) .collect(); + if !selected.is_empty() { + info!( + ?reason, + agents = selected.len(), + "Giving up agents whose shard has moved" + ); + } + futures::future::join_all(selected.into_iter().map(|worker| { let reason = reason.clone(); async move { worker.relinquish(reason).await } diff --git a/golem-worker-executor/src/services/oplog/mod.rs b/golem-worker-executor/src/services/oplog/mod.rs index 01f6b0c322..f75a3babff 100644 --- a/golem-worker-executor/src/services/oplog/mod.rs +++ b/golem-worker-executor/src/services/oplog/mod.rs @@ -489,9 +489,6 @@ pub type ReservedRawStartBuilder = pub type IndexedReservedStartBuilder = Box Result<(Vec, ReservedRawStartBuilder), String> + Send>; -/// A single oplog append that has already been synchronously enqueued in the oplog's ordering -/// domain. Creating this receipt reserves the entry's position; awaiting it returns the assigned -/// index after the append finishes. /// Why an oplog write was refused by the storage: the shard epoch this executor asserted is /// behind the one recorded for the oplog, because another executor owns the shard now. #[derive(Clone, Debug, PartialEq, Eq)] @@ -501,6 +498,16 @@ pub struct OplogFence { pub actual_epoch: Option, } +/// Told of every refusal the storage returns, carrying the epoch recorded on the oplog. +/// +/// That epoch is evidence of a generation somebody held for the agent's shard, which a shard +/// manager whose state lost history no longer knows about. The same refusal can be reported more +/// than once - a refused create, and then the refused open behind it - so an observer merges what +/// it is told rather than counting it. +pub trait OplogFenceObserver: Send + Sync { + fn fenced(&self, fence: &OplogFence); +} + /// The one way an oplog write can fail without taking the executor down. /// /// A `Fenced` write is not a storage failure - the storage is healthy and refused the write on @@ -554,6 +561,9 @@ impl Display for OplogError { impl std::error::Error for OplogError {} +/// A single oplog append that has already been synchronously enqueued in the oplog's ordering +/// domain. Creating this receipt reserves the entry's position; awaiting it returns the assigned +/// index after the append finishes. pub type OplogAddReceipt = BoxFuture<'static, Result>; #[derive(Clone, Debug, PartialEq, Eq)] @@ -1143,14 +1153,17 @@ struct OpenOplogEntry { /// Identifies this insertion, so that the remover the oplog runs when it is dropped removes /// this entry and not a replacement cached under the same agent after it. pub token: Arc<()>, + /// The epoch the opener that constructed this handle asked it to assert. + pub requested_epoch: Option, } impl OpenOplogEntry { - pub fn new(oplog: Arc, token: Arc<()>) -> Self { + pub fn new(oplog: Arc, token: Arc<()>, requested_epoch: Option) -> Self { Self { oplog: Arc::downgrade(&oplog), initial: Arc::new(AtomicBool::new(true)), token, + requested_epoch, } } } @@ -1177,6 +1190,7 @@ impl OpenOplogs { agent_id: &AgentId, constructor: impl OplogConstructor + 'static, ) -> Arc { + let requested_epoch = constructor.shard_epoch(); loop { let constructor_clone = constructor.clone(); let token = Arc::new(()); @@ -1206,7 +1220,7 @@ impl OpenOplogs { Arc::increment_strong_count(ptr); Arc::from_raw(ptr) }; - Ok(OpenOplogEntry::new(result, entry_token)) + Ok(OpenOplogEntry::new(result, entry_token, requested_epoch)) }, ) .await @@ -1231,7 +1245,21 @@ impl OpenOplogs { // Only a cache hit is discarded. An oplog refused at open is born fenced, and that // is what its opener asked for: it is handed back so that its writes are refused, // rather than constructed again and again. - if !just_constructed && oplog.fence().is_some() { + // + // Nor is a handle opened for an older ownership generation handed to an opener + // that asserts a newer epoch, fenced or not: a fork's unfenced copy, or a handle + // still held when the shard left this executor and came back at a higher epoch. + // It would go on writing at the epoch it was opened with, and the newer claim + // would never be recorded. Only a strictly newer request evicts. An equal or older + // one is handed the cached handle, so no second live handle is ever built at the + // epoch a handle already asserts. A handle that does not assert the epoch it was + // opened with (an ephemeral one) belongs to no generation: opened with an epoch, + // it is reused whatever epoch is requested. An evicted handle keeps any background + // work it started, such as a layered oplog's archive transfer, until its holder + // drops it. + let older_generation = requested_epoch > entry.requested_epoch + && oplog.shard_epoch() == entry.requested_epoch; + if !just_constructed && (oplog.fence().is_some() || older_generation) { // Removed by its own token, so a replacement cached meanwhile is left alone. self.oplogs .remove_if_cached(agent_id, |cached| { @@ -1259,4 +1287,10 @@ impl Debug for OpenOplogs { #[async_trait] pub trait OplogConstructor: Clone + Send { async fn create_oplog(self, close: Box) -> Arc; + + /// The epoch the oplog this constructor builds is asked to assert, or `None` for one opened + /// without an ownership claim. The open-oplog cache compares it with the epoch a cached + /// handle was opened with, so it has no default: a layer that left it out would hand an + /// older generation's handle to every newer opener. + fn shard_epoch(&self) -> Option; } diff --git a/golem-worker-executor/src/services/oplog/multilayer.rs b/golem-worker-executor/src/services/oplog/multilayer.rs index b22c04a74e..a949ed0a2d 100644 --- a/golem-worker-executor/src/services/oplog/multilayer.rs +++ b/golem-worker-executor/src/services/oplog/multilayer.rs @@ -303,15 +303,22 @@ impl MultiLayerOplogService { .remove(agent_id) .and_then(|transfer_fiber| transfer_fiber.upgrade()); - let transfer_fiber = transfer_fiber.and_then(|transfer_fiber| { + if let Some(transfer_fiber) = transfer_fiber { + Self::cancel_and_join_transfer(&transfer_fiber).await; + } + } + + /// Cancels a transfer fiber and returns once its task is gone, so it can start no more work. + async fn cancel_and_join_transfer(transfer_fiber: &TransferFiber) { + let transfer = { let mut transfer_fiber = transfer_fiber.lock().unwrap(); transfer_fiber.cancelled = true; transfer_fiber.transfer_fiber.take() - }); + }; - if let Some(transfer_fiber) = transfer_fiber { - transfer_fiber.abort(); - let _ = transfer_fiber.await; + if let Some(transfer) = transfer { + transfer.abort(); + let _ = transfer.await; } } @@ -423,6 +430,10 @@ impl CreateOplogConstructor { #[async_trait] impl OplogConstructor for CreateOplogConstructor { + fn shard_epoch(&self) -> Option { + self.shard_epoch + } + async fn create_oplog(self, close: Box) -> Arc { let agent_mode = self.agent_mode; let last_oplog_index = match self.last_oplog_index { @@ -1030,6 +1041,24 @@ impl MultiLayerOplog { Some(Self::archive(this, true).await) } + /// Ends this handle's background transfer for good and returns once nothing the transfer + /// started is still running. Does nothing for an oplog without archive layers. + /// + /// Dropping the handle is not enough: a transfer under way holds its own reference to it, and + /// goes on to drop the primary's prefix, deleting the primary oplog when that empties it, no + /// matter who has opened the agent's oplog since. The entries it did not move stay in the + /// primary layer, for the next handle to archive. + pub async fn try_abort_transfer(this: &Arc) { + let Some(this) = downcast_oplog::(this) else { + return; + }; + MultiLayerOplogService::cancel_and_join_transfer(&this.transfer_fiber).await; + // A transfer cancelled while it waited for its `drop_prefix` reply has already handed the + // job to the primary's actor, which runs it regardless. The actor serves jobs in the order + // they were sent, so a reply to a job sent after it means that job has finished. + this.primary.current_oplog_index().await; + } + async fn archive(this: Arc, blocking: bool) -> bool { let (done_tx, done_rx) = if blocking { let (done_tx, done_rx) = tokio::sync::oneshot::channel(); diff --git a/golem-worker-executor/src/services/oplog/plugin.rs b/golem-worker-executor/src/services/oplog/plugin.rs index c024d05e77..a2ea2b63e3 100644 --- a/golem-worker-executor/src/services/oplog/plugin.rs +++ b/golem-worker-executor/src/services/oplog/plugin.rs @@ -562,15 +562,11 @@ impl CreateOplogConstructor { #[async_trait] impl OplogConstructor for CreateOplogConstructor { + fn shard_epoch(&self) -> Option { + self.shard_epoch + } + async fn create_oplog(self, close: Box) -> Arc { - let last_oplog_index = match self.last_oplog_index { - Some(idx) => idx, - None => { - self.inner - .get_last_index(&self.owned_agent_id, self.agent_mode) - .await - } - }; let inner = if let Some(initial_entry) = self.initial_entry { if self.fresh { self.inner @@ -602,7 +598,7 @@ impl OplogConstructor for CreateOplogConstructor { .open( &self.owned_agent_id, self.agent_mode, - Some(last_oplog_index), + self.last_oplog_index, self.initial_worker_metadata.clone(), self.last_known_status.clone(), self.execution_status.clone(), @@ -610,6 +606,13 @@ impl OplogConstructor for CreateOplogConstructor { ) .await }; + // Taken from the opened inner oplog, not read up front: the inner open claims the shard + // epoch, so an index read before it could be behind a losing executor's last commit, and + // the forwarding buffer would then label its entries with indexes already in use. + let last_oplog_index = match self.last_oplog_index { + Some(idx) => idx, + None => inner.current_oplog_index().await, + }; Arc::new( ForwardingOplog::new( @@ -1454,6 +1457,14 @@ impl ForwardingOplogState { /// Complete ranges are read from the in-memory suffix when available; all other /// ranges are read whole from the persisted oplog without splicing sources. pub async fn try_flush(&mut self) { + // A fenced oplog is finished: nothing may be sent or checkpointed from it again. Without + // this, every tick and threshold flush picks the same batch and writes its checkpoint + // again, taking an index on an oplog that refuses the commit. The bookkeeping below is + // skipped as well, so the mirrored buffer is no longer pruned and the commit count no + // longer reset; both last only until the fenced worker is given up. + if self.inner.fence().is_some() { + return; + } let status = self.last_known_status.get(); let flush_set = self.reconcile_plugin_state(&status); @@ -1888,6 +1899,12 @@ impl ForwardingOplogState { sending_up_to: OplogIndex, last_batch_start: OplogIndex, ) -> Result<(), OplogFence> { + // Checked here as well as when the flush starts: the worker's own write can latch the + // fence while the flush awaits a send or a lookup, and a checkpoint added after that + // would still take an index for an entry that is never committed. + if let Some(fence) = self.inner.fence() { + return Err(fence); + } let checkpoint = OplogEntry::OplogProcessorCheckpoint { timestamp: golem_common::model::Timestamp::now_utc(), plugin_grant_id: grant_id, @@ -1922,8 +1939,9 @@ impl ForwardingOplogState { Ok(()) } - /// Logs a fenced checkpoint once and hands the fence back to the caller, which stops - /// forwarding for this agent. + /// Logs the checkpoint that found the fence and hands the fence back to the caller, which + /// stops forwarding for this agent. Every flush after it sees the latched fence and returns + /// before writing anything. fn stop_forwarding(&self, fence: OplogFence) -> OplogFence { tracing::info!( source_agent = %self.initial_worker_metadata.agent_id, @@ -1982,6 +2000,10 @@ impl ForwardingOplogState { /// delivery always goes to the recorded `target_agent_id` with deterministic /// idempotency keys. async fn try_locality_recovery(&mut self) { + // A migration is recorded with a checkpoint, which a fenced oplog no longer takes. + if self.inner.fence().is_some() { + return; + } let status = self.last_known_status.get(); // Ensure plugin_state is reconciled with current status self.reconcile_plugin_state(&status); @@ -2568,6 +2590,10 @@ mod tests { committed_idx: std::sync::Mutex, read_exact_count: std::sync::atomic::AtomicUsize, read_exact_requests: std::sync::Mutex>, + /// Refuses the next `commit` as fenced once set. The refusal latches, as it does on the + /// primary oplog, so every commit after it is refused and `fence` reports it. + armed_fence: std::sync::Mutex>, + latched_fence: std::sync::OnceLock, } #[allow(dead_code)] @@ -2579,9 +2605,15 @@ mod tests { committed_idx: std::sync::Mutex::new(OplogIndex::NONE), read_exact_count: std::sync::atomic::AtomicUsize::new(0), read_exact_requests: std::sync::Mutex::new(Vec::new()), + armed_fence: std::sync::Mutex::new(None), + latched_fence: std::sync::OnceLock::new(), } } + fn arm_fence(&self, fence: OplogFence) { + *self.armed_fence.lock().unwrap() = Some(fence); + } + fn read_exact_count(&self) -> usize { self.read_exact_count .load(std::sync::atomic::Ordering::Relaxed) @@ -2682,6 +2714,13 @@ mod tests { &self, _level: CommitLevel, ) -> Result, crate::services::oplog::OplogError> { + if let Some(fence) = self.latched_fence.get() { + return Err(OplogError::Fenced(fence.clone())); + } + if let Some(fence) = self.armed_fence.lock().unwrap().take() { + let _ = self.latched_fence.set(fence.clone()); + return Err(OplogError::Fenced(fence)); + } let entries = self.entries.lock().unwrap(); let current = *self.current_idx.lock().unwrap(); let mut committed = self.committed_idx.lock().unwrap(); @@ -2748,6 +2787,10 @@ mod tests { ) -> Result, String> { unimplemented!() } + + fn fence(&self) -> Option { + self.latched_fence.get().cloned() + } } fn test_worker_metadata( @@ -2855,6 +2898,87 @@ mod tests { ); } + // -------------------------------------------------------------------------- + // A fenced checkpoint ends forwarding: the flushes after it write and send nothing + // -------------------------------------------------------------------------- + + #[test] + async fn a_fenced_checkpoint_stops_later_flushes_before_they_write() { + let grant_id = EnvironmentPluginGrantId::new(); + let (metadata, status_lock) = test_worker_metadata(HashSet::from([grant_id])); + let recording_plugin = Arc::new(RecordingOplogProcessorPlugin::new()); + let components: Arc = Arc::new( + FakeComponentService::with_one_oplog_processor_plugin(grant_id), + ); + let in_memory = Arc::new(InMemoryOplog::new()); + in_memory.add(grow_memory(1)).await.unwrap(); + in_memory.add(grow_memory(2)).await.unwrap(); + in_memory.commit(CommitLevel::Always).await.unwrap(); + in_memory.arm_fence(OplogFence { + agent_id: metadata.agent_id.clone(), + expected_epoch: ShardEpoch(1), + actual_epoch: Some(ShardEpoch(2)), + }); + let inner: Arc = in_memory.clone(); + + // No target yet, so the first write is the checkpoint recording the resolved target. It + // is written before the grant is marked as sending, so nothing but the fence stops the + // next flush from resolving and writing it again. + let mut state = ForwardingOplogState { + buffer: None, + buffer_start_idx: OplogIndex::from_u64(3), + commit_count: 0, + last_send: Instant::now(), + oplog_plugins: recording_plugin.clone(), + initial_worker_metadata: metadata, + last_known_status: status_lock, + last_oplog_idx: OplogIndex::from_u64(2), + last_committed_idx: OplogIndex::from_u64(2), + components, + inner, + plugin_state: HashMap::from([( + grant_id, + LivePluginState { + target_agent_id: None, + confirmed_up_to: OplogIndex::NONE, + sending_up_to: OplogIndex::NONE, + send_in_progress: false, + last_batch_start: OplogIndex::NONE, + }, + )]), + pending_direct_commits: BTreeMap::new(), + worker_event_service: None, + monitor_tasks: Vec::new(), + }; + + state.try_flush().await; + assert_eq!(recording_plugin.send_count().await, 0); + assert_eq!( + in_memory.length().await, + 3, + "the refused checkpoint is the only entry the first flush adds" + ); + let last_oplog_idx = state.last_oplog_idx; + + state.try_flush().await; + state.try_flush().await; + + assert_eq!( + in_memory.length().await, + 3, + "a flush after the fence must not add another checkpoint" + ); + assert_eq!(state.last_oplog_idx, last_oplog_idx); + assert_eq!(recording_plugin.send_count().await, 0); + assert_eq!( + recording_plugin + .resolve_count + .load(std::sync::atomic::Ordering::Relaxed), + 1, + "a flush after the fence must not resolve the target again" + ); + } + // -------------------------------------------------------------------------- // U5 (partial): No active plugins → no send even with entries in buffer // -------------------------------------------------------------------------- diff --git a/golem-worker-executor/src/services/oplog/primary.rs b/golem-worker-executor/src/services/oplog/primary.rs index bbd963c4e6..1de0b391b8 100644 --- a/golem-worker-executor/src/services/oplog/primary.rs +++ b/golem-worker-executor/src/services/oplog/primary.rs @@ -23,9 +23,9 @@ use crate::services::oplog::reader::{ }; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, OpenOplogs, Oplog, - OplogAddReceipt, OplogConstructor, OplogError, OplogFence, OplogService, OrderedOplogStart, - PendingUpload, ReservedPayload, ReservedRawStartBuilder, cursor_value, next_scan_cursor, - scan_modes, + OplogAddReceipt, OplogConstructor, OplogError, OplogFence, OplogFenceObserver, OplogService, + OrderedOplogStart, PendingUpload, ReservedPayload, ReservedRawStartBuilder, cursor_value, + next_scan_cursor, scan_modes, }; use crate::storage::indexed::{ IndexedStorage, IndexedStorageError, IndexedStorageLabelledApi, IndexedStorageMetaNamespace, @@ -299,10 +299,15 @@ async fn retry_oplog_append( /// Records the epoch this executor is allowed to write `key` with, and reports the fence when /// the stored record is already ahead of it. /// -/// Monotonic on the storage side, so a re-grant at a higher epoch takes the oplog over while an -/// executor holding a stale one cannot claim it back. Written before the oplog's first entry - +/// Monotonic on the storage side once a record exists, so a re-grant at a higher epoch takes the +/// oplog over while an executor holding a stale one cannot claim it back. An oplog with no record +/// (new, deleted, or from before the record existed) is claimed by whichever epoch opens it +/// first. Written before the oplog's first entry - /// an absent record fences too, which is what closes the window between creating an oplog and /// recording who owns it. +/// +/// A refusal is also handed to `fence_observer`, because the epoch it carries is what a shard +/// manager whose state lost history has to mint above. async fn record_owning_epoch( indexed_storage: &(dyn IndexedStorage + Send + Sync), retry_config: &RetryConfig, @@ -310,6 +315,7 @@ async fn record_owning_epoch( agent_mode: AgentMode, key: &str, shard_epoch: ShardEpoch, + fence_observer: Option<&dyn OplogFenceObserver>, ) -> Option { let outcome = retry_storage_op_fenceable(retry_config, "upsert_oplog_metadata", key, || { let ns = IndexedStorageNamespace::OpLog { @@ -328,11 +334,23 @@ async fn record_owning_epoch( Ok(()) => None, Err(IndexedStorageError::Fenced { expected, actual, .. - }) => Some(OplogFence { - agent_id: owned_agent_id.agent_id(), - expected_epoch: expected, - actual_epoch: actual, - }), + }) => { + warn!( + agent_id = %owned_agent_id, + expected_epoch = expected.0, + actual_epoch = ?actual.map(|epoch| epoch.0), + "Oplog opened at a stale shard epoch: the shard has a new owner" + ); + let fence = OplogFence { + agent_id: owned_agent_id.agent_id(), + expected_epoch: expected, + actual_epoch: actual, + }; + if let Some(observer) = fence_observer { + observer.fenced(&fence); + } + Some(fence) + } // `retry_storage_op_fenceable` panics on every other permanent failure. Err(other) => unreachable!("unexpected storage error: {other}"), } @@ -367,7 +385,7 @@ async fn read_persisted_oplog_entries( /// /// Stores and retrieves individual oplog entries from the `IndexedStorage` implementation configured for /// the executor. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct PrimaryOplogService { indexed_storage: Arc, blob_storage: Arc, @@ -378,6 +396,32 @@ pub struct PrimaryOplogService { retry_config: RetryConfig, oplogs: OpenOplogs, stream_session_index: Arc>>, + /// Told of every refusal the storage returns for an oplog this service opened, so the epochs + /// the refusals carry can reach the shard manager. `None` reports nothing. + fence_observer: Option>, +} + +impl Debug for PrimaryOplogService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PrimaryOplogService") + .field("indexed_storage", &self.indexed_storage) + .field("blob_storage", &self.blob_storage) + .field("replicas", &self.replicas) + .field( + "max_operations_before_commit", + &self.max_operations_before_commit, + ) + .field( + "max_operations_before_commit_ephemeral", + &self.max_operations_before_commit_ephemeral, + ) + .field("max_payload_size", &self.max_payload_size) + .field("retry_config", &self.retry_config) + .field("oplogs", &self.oplogs) + .field("stream_session_index", &self.stream_session_index) + .field("fence_observer", &self.fence_observer.is_some()) + .finish() + } } impl PrimaryOplogService { @@ -404,9 +448,17 @@ impl PrimaryOplogService { retry_config, oplogs: OpenOplogs::new("primary oplog"), stream_session_index: Arc::new(std::sync::OnceLock::new()), + fence_observer: None, } } + /// Reports every refusal the storage returns for an oplog this service opens to `observer`, + /// with the epoch recorded on the oplog. + pub fn with_fence_observer(mut self, observer: Arc) -> Self { + self.fence_observer = Some(observer); + self + } + fn oplog_key(agent_id: &AgentId) -> String { agent_id.to_redis_key() } @@ -457,6 +509,47 @@ impl PrimaryOplogService { }); } + async fn open_with( + &self, + owned_agent_id: &OwnedAgentId, + agent_mode: AgentMode, + last_oplog_index: Option, + initial_worker_metadata: AgentMetadata, + shard_epoch: Option, + reconcile_last_index: bool, + ) -> Arc { + record_oplog_call("open"); + + let key = Self::oplog_key(&owned_agent_id.agent_id); + let max_operations_before_commit = match agent_mode { + AgentMode::Durable => self.max_operations_before_commit, + AgentMode::Ephemeral => self.max_operations_before_commit_ephemeral, + }; + + self.oplogs + .get_or_open( + &owned_agent_id.agent_id, + CreateOplogConstructor::new( + shard_epoch, + self.indexed_storage.clone(), + self.blob_storage.clone(), + self.replicas, + max_operations_before_commit, + self.max_payload_size, + self.retry_config.clone(), + key, + last_oplog_index, + reconcile_last_index, + owned_agent_id.clone(), + agent_mode, + initial_worker_metadata.created_by, + self.stream_session_index(), + self.fence_observer.clone(), + ), + ) + .await + } + async fn get_last_index_from_storage( indexed_storage: &(dyn IndexedStorage + Send + Sync), owned_agent_id: &OwnedAgentId, @@ -575,36 +668,22 @@ impl OplogService for PrimaryOplogService { agent_mode: AgentMode, initial_entry: OplogEntry, initial_worker_metadata: AgentMetadata, - last_known_status: read_only_lock::arc_swap::ReadOnlyView, - execution_status: read_only_lock::std::ReadOnlyLock, + _last_known_status: read_only_lock::arc_swap::ReadOnlyView, + _execution_status: read_only_lock::std::ReadOnlyLock, shard_epoch: Option, ) -> Arc { record_oplog_call("create"); let key = Self::oplog_key(&owned_agent_id.agent_id); - let already_exists: bool = { - let is = self.indexed_storage.clone(); - let agent_id = owned_agent_id.agent_id(); - let key = key.clone(); - retry_storage_op(&self.retry_config, "create_exists", &key, || { - let is = is.clone(); - let ns = IndexedStorageNamespace::OpLog { - agent_id: agent_id.clone(), - agent_mode, - }; - let key = key.clone(); - async move { is.with("oplog", "create").exists(ns, &key).await } - }) - .await - }; - if already_exists { - panic!("oplog for worker {owned_agent_id} already exists in indexed storage") - } - - // The record goes in before the first entry. If it is refused, this executor has already - // lost the shard: skip the append entirely and let `open` below hand back an oplog that - // refuses every write. + // The record goes in before the existence probe and the first entry. A probe taken before + // the claim can miss a `Create` that an executor at an older epoch lands in between, and + // the initial append would then collide with it. If the claim is refused, this executor + // has already lost the shard: it writes nothing, so whether the owner created the oplog + // first is not its question, and `open` below hands back an oplog that refuses every write. + // The refusal is reported here even though the open behind it may report it again: an + // unfenced handle this service still holds at the same epoch is handed back without + // asking the storage, and then this is the only refusal before a write. let fenced_at_create = match shard_epoch { Some(epoch) => record_owning_epoch( &*self.indexed_storage, @@ -613,6 +692,7 @@ impl OplogService for PrimaryOplogService { agent_mode, &key, epoch, + self.fence_observer.as_deref(), ) .await .is_some(), @@ -620,6 +700,26 @@ impl OplogService for PrimaryOplogService { }; if !fenced_at_create { + let already_exists: bool = { + let is = self.indexed_storage.clone(); + let agent_id = owned_agent_id.agent_id(); + let key = key.clone(); + retry_storage_op(&self.retry_config, "create_exists", &key, || { + let is = is.clone(); + let ns = IndexedStorageNamespace::OpLog { + agent_id: agent_id.clone(), + agent_mode, + }; + let key = key.clone(); + async move { is.with("oplog", "create").exists(ns, &key).await } + }) + .await + }; + + if already_exists { + panic!("oplog for worker {owned_agent_id} already exists in indexed storage") + } + self.append_initial_entry( owned_agent_id, agent_mode, @@ -631,14 +731,14 @@ impl OplogService for PrimaryOplogService { .await; } - self.open( + // The claim came before the initial entry, so `INITIAL` is exact and needs no re-read. + self.open_with( owned_agent_id, agent_mode, Some(OplogIndex::INITIAL), initial_worker_metadata, - last_known_status, - execution_status, shard_epoch, + false, ) .await } @@ -649,8 +749,8 @@ impl OplogService for PrimaryOplogService { agent_mode: AgentMode, initial_entry: OplogEntry, initial_worker_metadata: AgentMetadata, - last_known_status: read_only_lock::arc_swap::ReadOnlyView, - execution_status: read_only_lock::std::ReadOnlyLock, + _last_known_status: read_only_lock::arc_swap::ReadOnlyView, + _execution_status: read_only_lock::std::ReadOnlyLock, shard_epoch: Option, ) -> Arc { record_oplog_call("create_fresh"); @@ -668,6 +768,7 @@ impl OplogService for PrimaryOplogService { agent_mode, &key, epoch, + self.fence_observer.as_deref(), ) .await .is_some(), @@ -686,14 +787,15 @@ impl OplogService for PrimaryOplogService { .await; } - self.open( + // Claimed before the initial entry, so `INITIAL` is exact; not re-reading it keeps a fresh + // create free of storage reads. + self.open_with( owned_agent_id, agent_mode, Some(OplogIndex::INITIAL), initial_worker_metadata, - last_known_status, - execution_status, shard_epoch, + false, ) .await } @@ -708,34 +810,17 @@ impl OplogService for PrimaryOplogService { _execution_status: read_only_lock::std::ReadOnlyLock, shard_epoch: Option, ) -> Arc { - record_oplog_call("open"); - - let key = Self::oplog_key(&owned_agent_id.agent_id); - let max_operations_before_commit = match agent_mode { - AgentMode::Durable => self.max_operations_before_commit, - AgentMode::Ephemeral => self.max_operations_before_commit_ephemeral, - }; - - self.oplogs - .get_or_open( - &owned_agent_id.agent_id, - CreateOplogConstructor::new( - shard_epoch, - self.indexed_storage.clone(), - self.blob_storage.clone(), - self.replicas, - max_operations_before_commit, - self.max_payload_size, - self.retry_config.clone(), - key, - last_oplog_index, - owned_agent_id.clone(), - agent_mode, - initial_worker_metadata.created_by, - self.stream_session_index(), - ), - ) - .await + // An index handed in by a caller was read before this open claims the epoch. + let reconcile_last_index = last_oplog_index.is_some(); + self.open_with( + owned_agent_id, + agent_mode, + last_oplog_index, + initial_worker_metadata, + shard_epoch, + reconcile_last_index, + ) + .await } async fn get_last_index( @@ -948,11 +1033,15 @@ struct CreateOplogConstructor { retry_config: RetryConfig, key: String, last_oplog_idx: Option, + /// `last_oplog_idx` was read before this constructor claims the epoch, so it may be behind + /// entries an executor at an older epoch committed in between. + reconcile_last_index: bool, owned_agent_id: OwnedAgentId, agent_mode: AgentMode, account_id: AccountId, stream_session_index: Option>, shard_epoch: Option, + fence_observer: Option>, } impl CreateOplogConstructor { @@ -967,10 +1056,12 @@ impl CreateOplogConstructor { retry_config: RetryConfig, key: String, last_oplog_idx: Option, + reconcile_last_index: bool, owned_agent_id: OwnedAgentId, agent_mode: AgentMode, account_id: AccountId, stream_session_index: Option>, + fence_observer: Option>, ) -> Self { Self { shard_epoch, @@ -982,29 +1073,23 @@ impl CreateOplogConstructor { retry_config, key, last_oplog_idx, + reconcile_last_index, owned_agent_id, agent_mode, account_id, stream_session_index, + fence_observer, } } } #[async_trait] impl OplogConstructor for CreateOplogConstructor { + fn shard_epoch(&self) -> Option { + self.shard_epoch + } + async fn create_oplog(self, close: Box) -> Arc { - let last_oplog_idx = match self.last_oplog_idx { - Some(idx) => idx, - None => { - PrimaryOplogService::get_last_index_from_storage( - &*self.indexed_storage, - &self.owned_agent_id, - self.agent_mode, - &self.retry_config, - ) - .await - } - }; // Recorded before the oplog is usable, so an executor whose shard has moved is refused // at its very first write rather than after replaying the new owner's entries. let fence = match self.shard_epoch { @@ -1016,12 +1101,37 @@ impl OplogConstructor for CreateOplogConstructor { self.agent_mode, &self.key, shard_epoch, + self.fence_observer.as_deref(), ) .await } None => None, }; + // Read after the claim: once it returns, every writer at an older epoch is refused, so the + // read sees every entry that will ever precede this handle's first write. An index a + // caller read before the claim can be behind by whatever a losing executor committed in + // between, and this handle's first append would collide with it. That index is merged + // rather than replaced, because it may count entries this layer no longer holds, such as + // ones already moved to an archive. + let stored_last_index = || { + PrimaryOplogService::get_last_index_from_storage( + &*self.indexed_storage, + &self.owned_agent_id, + self.agent_mode, + &self.retry_config, + ) + }; + let last_oplog_idx = match self.last_oplog_idx { + None => stored_last_index().await, + Some(idx) + if self.reconcile_last_index && self.shard_epoch.is_some() && fence.is_none() => + { + OplogIndex::from_u64(idx.as_u64().max(stored_last_index().await.as_u64())) + } + Some(idx) => idx, + }; + Arc::new(PrimaryOplog::new( self.shard_epoch, fence, @@ -1037,6 +1147,7 @@ impl OplogConstructor for CreateOplogConstructor { self.agent_mode, self.account_id, self.stream_session_index, + self.fence_observer, close, )) } @@ -1198,6 +1309,7 @@ impl PrimaryOplog { agent_mode: AgentMode, account_id: AccountId, stream_session_index: Option>, + fence_observer: Option>, close: Box, ) -> Self { let account_id_label = account_id.to_string(); @@ -1209,6 +1321,7 @@ impl PrimaryOplog { let mut state = PrimaryOplogState { shard_epoch, fence: fence.clone(), + fence_observer, indexed_storage, blob_storage, replicas, @@ -1384,9 +1497,11 @@ impl PrimaryOplog { let _ = done.send(result); } OplogJob::Flush { done } => { - // The caller gets no error to act on. A fence is latched on the state, so - // every later write fails on it without asking the storage again; a - // transient storage failure is fatal here as everywhere else. + // The job has no error to reply with. A fence is latched on the state, + // where `wait_for_replicas` reads it after this job, so a fenced flush is + // reported as not durable rather than lost; every subsequent write fails on + // it without asking the storage again. A transient storage failure is fatal + // here as everywhere else. match state.commit(CommitLevel::Always).await { Ok(_) | Err(OplogError::Fenced(_)) => {} Err(error) => panic!("oplog write: {error}"), @@ -1692,6 +1807,9 @@ struct PrimaryOplogState { /// now, so there is nothing to be gained by asking the storage again. Shared with the handle, /// which answers [`Oplog::fence`] from it without a round trip through the actor. fence: Arc>, + /// Told of the refusal that sets [`Self::fence`]; the latched fast-fail asks the storage + /// nothing and reports nothing. + fence_observer: Option>, } impl PrimaryOplogState { @@ -1834,9 +1952,23 @@ impl PrimaryOplogState { .map_err(|err| Self::as_oplog_error(&self.owned_agent_id, err)) .inspect_err(|err| { if let OplogError::Fenced(fence) = err { - let _ = self.fence.set(fence.clone()); - // The drained entries are dropped: this oplog is not ours to write. - self.pending_uploads.clear(); + // The drained entries are dropped: this oplog is not ours to write. Nothing else + // is undone - the commit barrier above already awaited every payload the batch + // referenced, so those blobs are durable and stay behind with no entry pointing at + // them. The epoch the storage holds is reported, so a shard manager whose state + // lost history can mint above it. Warned only when it latches: each write after + // that fails fast on the latch without reaching the storage. + if self.fence.set(fence.clone()).is_ok() { + warn!( + agent_id = %self.owned_agent_id, + expected_epoch = fence.expected_epoch.0, + actual_epoch = ?fence.actual_epoch.map(|epoch| epoch.0), + "Oplog append fenced: the shard has a new owner, refusing further writes" + ); + } + if let Some(observer) = &self.fence_observer { + observer.fenced(fence); + } } })?; @@ -2116,6 +2248,11 @@ impl Oplog for PrimaryOplog { record_oplog_call("wait_for_replicas"); self.run_job(|done| OplogJob::Flush { done }).await; + // A refused flush reached no replica. The storage would still answer with its replica + // count, and passing that on would tell the caller that entries it turned away are durable. + if self.fence.get().is_some() { + return false; + } let reader = self.run_job(|done| OplogJob::Reader { done }).await; let replicas = replicas.min(reader.replicas); match reader diff --git a/golem-worker-executor/src/services/oplog/tests.rs b/golem-worker-executor/src/services/oplog/tests.rs index 2c30f444f0..a90e9ef058 100644 --- a/golem-worker-executor/src/services/oplog/tests.rs +++ b/golem-worker-executor/src/services/oplog/tests.rs @@ -539,6 +539,8 @@ pub(crate) struct ReadCountingIndexedStorage { append_many_attempts: AtomicUsize, append_many_batch_ptr: AtomicUsize, append_many_batch_changed: AtomicBool, + drop_prefix_started: StdMutex>>, + release_drop_prefix: Option>, } impl ReadCountingIndexedStorage { @@ -546,6 +548,15 @@ impl ReadCountingIndexedStorage { Self::default() } + /// Every `drop_prefix` waits for `release`; the first one signals `started` when it arrives. + fn blocking_drop_prefix(started: oneshot::Sender<()>, release: Arc) -> Self { + Self { + drop_prefix_started: StdMutex::new(Some(started)), + release_drop_prefix: Some(release), + ..Self::default() + } + } + fn discarding_compressed_appends() -> Self { Self { discard_compressed_appends: true, @@ -908,6 +919,13 @@ impl IndexedStorage for ReadCountingIndexedStorage { key: &str, last_dropped_id: u64, ) -> Result<(), IndexedStorageError> { + if let Some(release) = &self.release_drop_prefix { + let started = self.drop_prefix_started.lock().unwrap().take(); + if let Some(started) = started { + let _ = started.send(()); + } + release.notified().await; + } self.inner .drop_prefix(svc_name, api_name, namespace, key, last_dropped_id) .await @@ -7675,6 +7693,78 @@ async fn a_fenced_oplog_is_not_handed_out_again_while_it_is_still_held(_tracing: ); } +/// What every write that gates a side effect relies on: an add's answer is no evidence the entry +/// was written, only the latch read after the commit is. A below-threshold add on a moved shard +/// buffers and succeeds, before and after the fence latches, and the refused commit is what +/// latches it. +#[test] +async fn a_below_threshold_add_on_a_moved_shard_is_latched_by_the_refused_commit( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "moved".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let executor = fencing_oplog_service(&tempdir, "shared").await; + let other_executor = fencing_oplog_service(&tempdir, "shared").await; + let open = |service: &PrimaryOplogService, epoch: u64| { + let service = service.clone(); + let agent_id = agent_id.clone(); + let owned_agent_id = owned_agent_id.clone(); + async move { + service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(epoch)), + ) + .await + } + }; + + let stale = open(&executor, 8).await; + let owner = open(&other_executor, 9).await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + assert_eq!(stale.fence(), None, "nothing has been refused yet"); + + stale + .add(OplogEntry::exited().rounded()) + .await + .expect("a below-threshold add only buffers, so the moved shard does not refuse it"); + assert!(matches!( + stale.commit(CommitLevel::Always).await, + Err(OplogError::Fenced(_)) + )); + let fence = stale + .fence() + .expect("the refused commit must latch the fence before it returns"); + assert_eq!(fence.expected_epoch, ShardEpoch(8)); + assert_eq!(fence.actual_epoch, Some(ShardEpoch(9))); + + stale + .add(OplogEntry::exited().rounded()) + .await + .expect("a latched fence does not stop a below-threshold add"); + assert!(matches!( + stale.commit(CommitLevel::Always).await, + Err(OplogError::Fenced(_)) + )); + assert_eq!( + stale.fence(), + Some(fence), + "the latch keeps the first refusal" + ); +} + #[test] async fn an_invocation_started_records_the_epoch_its_oplog_asserts(_tracing: &Tracing) { let tempdir = tempfile::TempDir::new().unwrap(); @@ -7816,3 +7906,931 @@ async fn an_executor_that_loses_the_shard_mid_flight_is_refused_at_its_next_writ "the refused entry must not be there" ); } + +#[test] +async fn wait_for_replicas_does_not_report_a_fenced_flush_as_durable(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "flushed".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let losing_executor = fencing_oplog_service(&tempdir, "flushed").await; + let owning_executor = fencing_oplog_service(&tempdir, "flushed").await; + + let loser = losing_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(8)), + ) + .await; + let owner = owning_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(9)), + ) + .await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + + // The guest's `oplog-commit` path: the entry is only buffered, and the flush inside + // `wait_for_replicas` is what the storage refuses. The fencing backends have no replicas to + // wait for, so a count taken after the refusal would read as a successful commit. + loser.add(OplogEntry::exited().rounded()).await.unwrap(); + assert!( + !loser.wait_for_replicas(1, Duration::from_secs(1)).await, + "a flush the storage refused must not be reported as durable" + ); + match loser.fence() { + Some(fence) => assert_eq!( + fence.actual_epoch, + Some(golem_common::model::ShardEpoch(9)), + "the fence must name the epoch that owns the oplog now" + ), + None => panic!("the refused flush must latch the fence"), + } + assert_eq!( + owner.length().await, + 1, + "the losing executor must not have appended to the owner's oplog" + ); +} + +#[test] +async fn an_opener_at_a_newer_epoch_is_not_handed_the_older_generations_handle(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "came-back".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let executor = fencing_oplog_service(&tempdir, "came-back").await; + let open = |epoch: u64| { + let service = executor.clone(); + let agent_id = agent_id.clone(); + let owned_agent_id = owned_agent_id.clone(); + async move { + service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(epoch)), + ) + .await + } + }; + + // The shard left this executor at epoch 5 and came back at 7 while the epoch-5 handle is still + // held. Nothing has fenced that handle - nobody has written since - so only the epoch it was + // opened with tells the cache it belongs to the older generation. + let old = open(5).await; + let new = open(7).await; + assert!( + !Arc::ptr_eq(&new, &old), + "the opener at epoch 7 was handed the handle opened at 5" + ); + assert_eq!(new.shard_epoch(), Some(ShardEpoch(7))); + new.add(OplogEntry::suspend().rounded()).await.unwrap(); + new.commit(CommitLevel::Always).await.unwrap(); + + let write = async { + old.add(OplogEntry::exited().rounded()).await?; + old.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + match write { + Err(OplogError::Fenced(fence)) => { + assert_eq!(fence.expected_epoch, ShardEpoch(5)); + assert_eq!(fence.actual_epoch, Some(ShardEpoch(7))); + } + other => panic!("expected the older generation's write to be fenced, got {other:?}"), + } + + // An equal or older request is handed the current handle. Building another would put two + // live writers on epoch 7, and they would collide on the oplog's keys. + let again = open(7).await; + assert!( + Arc::ptr_eq(&again, &new), + "an opener at the same epoch must share the handle" + ); + let stale = open(5).await; + assert!( + Arc::ptr_eq(&stale, &new), + "an opener at an older epoch must not evict the newer handle" + ); +} + +#[test] +async fn an_ephemeral_handle_is_reused_whatever_epoch_is_requested(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let primary = Arc::new(fencing_oplog_service(&tempdir, "ephemeral").await); + let archive: Arc = Arc::new(CompressedOplogArchiveService::new( + Arc::new(InMemoryIndexedStorage::new()), + 1, + RetryConfig::default(), + )); + let service = MultiLayerOplogService::new(primary, nev![archive], 10, 10); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "ephemeral".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let mut metadata = make_agent_metadata(agent_id, account_id, environment_id); + metadata.agent_mode = AgentMode::Ephemeral; + let open = |epoch: u64| { + let service = service.clone(); + let owned_agent_id = owned_agent_id.clone(); + let metadata = metadata.clone(); + async move { + service + .open( + &owned_agent_id, + AgentMode::Ephemeral, + None, + metadata, + default_last_known_status(), + default_execution_status(AgentMode::Ephemeral), + Some(ShardEpoch(epoch)), + ) + .await + } + }; + + // An ephemeral handle asserts no epoch whatever it was opened with, so it belongs to no + // ownership generation and a newer request is no reason to rebuild it. + let first = open(5).await; + assert_eq!(first.shard_epoch(), None); + let second = open(7).await; + assert!( + Arc::ptr_eq(&second, &first), + "an ephemeral handle was rebuilt for a newer epoch" + ); +} + +#[test] +async fn a_fork_target_handle_is_not_reused_by_the_owners_first_open(_tracing: &Tracing) { + let tempdir = tempfile::TempDir::new().unwrap(); + let primary = Arc::new(fencing_oplog_service(&tempdir, "fork-target").await); + let archive: Arc = Arc::new(CompressedOplogArchiveService::new( + Arc::new(InMemoryIndexedStorage::new()), + 1, + RetryConfig::default(), + )); + // Through the layered service, as production opens it: each layer caches its own handle, and + // every one of them has to decline the fork's. + let service = MultiLayerOplogService::new(primary, nev![archive], 10, 10); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "fork-target".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let create_entry = OplogEntry::create( + agent_id.clone(), + AgentMode::Durable, + ComponentRevision::new(1).unwrap(), + Vec::new(), + environment_id, + account_id, + None, + 100, + 100, + HashSet::new(), + Vec::new(), + None, + Uuid::new_v4(), + ) + .rounded(); + + // The fork copies into the target through a handle that asserts no epoch, since the target's + // shard may belong to another executor. Here that handle is still held when the owner opens + // the target. + let forked = service + .create( + &owned_agent_id, + AgentMode::Durable, + create_entry, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + None, + ) + .await; + forked.add(OplogEntry::suspend().rounded()).await.unwrap(); + forked.commit(CommitLevel::Always).await.unwrap(); + + let owner = service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(4)), + ) + .await; + assert!( + !Arc::ptr_eq(&owner, &forked), + "the owner's first open was handed the fork's unfenced handle" + ); + assert_eq!(owner.shard_epoch(), Some(ShardEpoch(4))); + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + + // Another executor takes the shard at epoch 5. The owner's next write is refused only if its + // open recorded epoch 4; through the fork's handle it would have recorded nothing and been + // accepted. + let other_executor = fencing_oplog_service(&tempdir, "fork-target").await; + let other = other_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(5)), + ) + .await; + other.add(OplogEntry::suspend().rounded()).await.unwrap(); + other.commit(CommitLevel::Always).await.unwrap(); + + let write = async { + owner.add(OplogEntry::exited().rounded()).await?; + owner.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + match write { + Err(OplogError::Fenced(fence)) => { + assert_eq!(fence.expected_epoch, ShardEpoch(4)); + assert_eq!(fence.actual_epoch, Some(ShardEpoch(5))); + } + other => panic!("expected the owner's write to be fenced, got {other:?}"), + } +} + +#[test] +async fn an_owner_opening_on_a_stale_last_index_starts_after_the_losers_last_write( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "stale-last-index".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let owning_executor = fencing_oplog_service(&tempdir, "shared").await; + let losing_executor = fencing_oplog_service(&tempdir, "shared").await; + + let loser = losing_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(5)), + ) + .await; + loser.add(OplogEntry::suspend().rounded()).await.unwrap(); + loser.commit(CommitLevel::Always).await.unwrap(); + + // A layer above the primary reads the last index before the primary claims the epoch, as the + // layered service does, and the losing executor commits again in that window: the record + // still says 5, so the write is accepted. + let stale_last_index = owning_executor + .get_last_index(&owned_agent_id, AgentMode::Durable) + .await; + assert_eq!(stale_last_index, OplogIndex::from_u64(1)); + loser.add(OplogEntry::suspend().rounded()).await.unwrap(); + loser.commit(CommitLevel::Always).await.unwrap(); + + let owner = owning_executor + .open( + &owned_agent_id, + AgentMode::Durable, + Some(stale_last_index), + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(6)), + ) + .await; + // Starting from the stale index, this append would reuse the loser's id and fail-stop the + // executor that owns the shard. + owner.add(OplogEntry::exited().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + assert_eq!(owner.length().await, 3); + assert_eq!(owner.current_oplog_index().await, OplogIndex::from_u64(3)); + + let write = async { + loser.add(OplogEntry::exited().rounded()).await?; + loser.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + match write { + Err(OplogError::Fenced(fence)) => { + assert_eq!(fence.expected_epoch, ShardEpoch(5)); + assert_eq!(fence.actual_epoch, Some(ShardEpoch(6))); + } + other => panic!("expected the losing executor's write to be fenced, got {other:?}"), + } +} + +#[test] +async fn a_stale_create_of_an_oplog_the_owner_already_created_is_fenced_not_fatal( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "created-twice".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let owning_executor = fencing_oplog_service(&tempdir, "shared").await; + let losing_executor = fencing_oplog_service(&tempdir, "shared").await; + let create_entry = || { + OplogEntry::create( + agent_id.clone(), + AgentMode::Durable, + ComponentRevision::new(1).unwrap(), + Vec::new(), + environment_id, + account_id, + None, + 100, + 100, + HashSet::new(), + Vec::new(), + None, + Uuid::new_v4(), + ) + .rounded() + }; + + let owner = owning_executor + .create( + &owned_agent_id, + AgentMode::Durable, + create_entry(), + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(6)), + ) + .await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + + // An executor that lost the shard creates the same agent. Its claim is refused, so it writes + // nothing and must not fail-stop over an oplog that belongs to the owner. + let stale = losing_executor + .create( + &owned_agent_id, + AgentMode::Durable, + create_entry(), + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(5)), + ) + .await; + match stale.fence() { + Some(fence) => { + assert_eq!(fence.expected_epoch, ShardEpoch(5)); + assert_eq!(fence.actual_epoch, Some(ShardEpoch(6))); + } + None => panic!("the refused create must hand back a fenced oplog"), + } + let write = async { + stale.add(OplogEntry::exited().rounded()).await?; + stale.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + assert!( + matches!(write, Err(OplogError::Fenced(_))), + "expected the stale create's write to be fenced, got {write:?}" + ); + assert_eq!( + owner.length().await, + 2, + "the stale create must not have appended to the owner's oplog" + ); +} + +/// Keeps every fence it is told of, in order. +#[derive(Default)] +struct RecordingFenceObserver { + fences: StdMutex>, +} + +impl RecordingFenceObserver { + fn fences(&self) -> Vec { + self.fences.lock().unwrap().clone() + } +} + +impl OplogFenceObserver for RecordingFenceObserver { + fn fenced(&self, fence: &OplogFence) { + self.fences.lock().unwrap().push(fence.clone()); + } +} + +fn initial_create_entry( + agent_id: &AgentId, + environment_id: EnvironmentId, + account_id: AccountId, +) -> OplogEntry { + OplogEntry::create( + agent_id.clone(), + AgentMode::Durable, + ComponentRevision::new(1).unwrap(), + Vec::new(), + environment_id, + account_id, + None, + 100, + 100, + HashSet::new(), + Vec::new(), + None, + Uuid::new_v4(), + ) + .rounded() +} + +#[test] +async fn a_refused_open_or_create_reports_the_stored_epoch_to_the_fence_observer( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let recorder = Arc::new(RecordingFenceObserver::default()); + let owning_executor = fencing_oplog_service(&tempdir, "shared").await; + let losing_executor = fencing_oplog_service(&tempdir, "shared") + .await + .with_fence_observer(recorder.clone()); + let opened = AgentId { + component_id: ComponentId::new(), + agent_id: "opened-by-the-loser".into(), + }; + let created = AgentId { + component_id: ComponentId::new(), + agent_id: "created-by-the-loser".into(), + }; + let expected_fence = |agent_id: &AgentId| OplogFence { + agent_id: agent_id.clone(), + expected_epoch: ShardEpoch(5), + actual_epoch: Some(ShardEpoch(6)), + }; + + for agent_id in [&opened, &created] { + let owner = owning_executor + .create( + &OwnedAgentId::new(environment_id, agent_id), + AgentMode::Durable, + initial_create_entry(agent_id, environment_id, account_id), + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(6)), + ) + .await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + } + + let stale = losing_executor + .open( + &OwnedAgentId::new(environment_id, &opened), + AgentMode::Durable, + None, + make_agent_metadata(opened.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(5)), + ) + .await; + assert!(stale.fence().is_some(), "the stale open was not refused"); + let reported = recorder.fences(); + assert!( + !reported.is_empty(), + "the refused open reported nothing to the observer" + ); + for fence in &reported { + assert_eq!(fence, &expected_fence(&opened)); + } + + // Born fenced, so its writes fail on the latch without asking the storage again. + let write = async { + stale.add(OplogEntry::exited().rounded()).await?; + stale.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + assert!( + matches!(write, Err(OplogError::Fenced(_))), + "expected the stale open's write to be fenced, got {write:?}" + ); + assert_eq!( + recorder.fences().len(), + reported.len(), + "a write refused by the latch reported a refusal the storage never made" + ); + + // On a cache miss a refused create is reported twice, by `create` and by the open behind it, + // and the observer merges. So what is asserted is what was learned, not how often. + let reported_before_create = recorder.fences().len(); + let stale_create = losing_executor + .create( + &OwnedAgentId::new(environment_id, &created), + AgentMode::Durable, + initial_create_entry(&created, environment_id, account_id), + make_agent_metadata(created.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(5)), + ) + .await; + assert!( + stale_create.fence().is_some(), + "the stale create was not refused" + ); + let reported = recorder.fences().split_off(reported_before_create); + assert!( + !reported.is_empty(), + "the refused create reported nothing to the observer" + ); + for fence in &reported { + assert_eq!(fence, &expected_fence(&created)); + } +} + +#[test] +async fn a_create_refused_behind_a_cached_handle_still_reports_the_stored_epoch( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "created-again-while-held".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let recorder = Arc::new(RecordingFenceObserver::default()); + let owning_executor = fencing_oplog_service(&tempdir, "shared").await; + let losing_executor = fencing_oplog_service(&tempdir, "shared") + .await + .with_fence_observer(recorder.clone()); + let create_at_5 = || { + losing_executor.create( + &owned_agent_id, + AgentMode::Durable, + initial_create_entry(&agent_id, environment_id, account_id), + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(5)), + ) + }; + + // Created while this executor owned the shard, and held without a write. + let held = create_at_5().await; + assert!(held.fence().is_none()); + assert!(recorder.fences().is_empty()); + + // The shard moves, and its new owner claims the oplog. + let owner = owning_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(6)), + ) + .await; + assert!(owner.fence().is_none()); + + // The claim is refused, and the open behind it hands back the held handle without asking the + // storage, so the refusal `create` reports is the only one before a write. + let again = create_at_5().await; + assert!( + Arc::ptr_eq(&again, &held), + "the held handle was not handed back, so this is not the cache hit under test" + ); + assert_eq!( + recorder.fences(), + vec![OplogFence { + agent_id: agent_id.clone(), + expected_epoch: ShardEpoch(5), + actual_epoch: Some(ShardEpoch(6)), + }] + ); +} + +#[test] +async fn a_refused_append_reports_the_stored_epoch_and_the_latch_does_not_report_again( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "appended-by-the-loser".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let recorder = Arc::new(RecordingFenceObserver::default()); + let owning_executor = fencing_oplog_service(&tempdir, "shared").await; + let losing_executor = fencing_oplog_service(&tempdir, "shared") + .await + .with_fence_observer(recorder.clone()); + let open = |service: &PrimaryOplogService, epoch: u64| { + let service = service.clone(); + let agent_id = agent_id.clone(); + let owned_agent_id = owned_agent_id.clone(); + async move { + service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(ShardEpoch(epoch)), + ) + .await + } + }; + + let stale = open(&losing_executor, 5).await; + assert!(stale.fence().is_none()); + assert!(recorder.fences().is_empty()); + let owner = open(&owning_executor, 6).await; + + let write = async { + stale.add(OplogEntry::exited().rounded()).await?; + stale.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + }; + let refused = write.await; + assert!( + matches!(refused, Err(OplogError::Fenced(_))), + "expected the losing executor's write to be fenced, got {refused:?}" + ); + let expected = OplogFence { + agent_id: agent_id.clone(), + expected_epoch: ShardEpoch(5), + actual_epoch: Some(ShardEpoch(6)), + }; + assert_eq!( + recorder.fences(), + vec![expected.clone()], + "one refused append is one report" + ); + + let again = async { + stale.add(OplogEntry::exited().rounded()).await?; + stale.commit(CommitLevel::Always).await?; + Ok::<_, OplogError>(()) + } + .await; + assert!(matches!(again, Err(OplogError::Fenced(_)))); + assert_eq!( + recorder.fences(), + vec![expected], + "the latched fast-fail asked the storage nothing, so it must report nothing" + ); + + // The owner, with no observer, writes as before. + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); +} + +async fn open_unfenced_fork_target( + service: &MultiLayerOplogService, + owned_agent_id: &OwnedAgentId, +) -> Arc { + service + .open( + owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata( + owned_agent_id.agent_id.clone(), + AccountId::new(), + owned_agent_id.environment_id, + ), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + None, + ) + .await +} + +#[test] +async fn closing_a_fork_target_ends_the_archive_transfer_its_copy_scheduled(_tracing: &Tracing) { + let indexed_storage = Arc::new(InMemoryIndexedStorage::new()); + let blob_storage = Arc::new(InMemoryBlobStorage::new()); + let primary = Arc::new( + PrimaryOplogService::new( + indexed_storage.clone(), + blob_storage.clone(), + 1, + 1, + 100, + RetryConfig::default(), + ) + .await, + ); + let (append_started_tx, append_started_rx) = oneshot::channel(); + let release_append = Arc::new(Notify::new()); + let append_finished = Arc::new(Notify::new()); + let archive: Arc = Arc::new(BlockingArchiveService { + inner: Arc::new(CompressedOplogArchiveService::new( + indexed_storage.clone(), + 1, + RetryConfig::default(), + )), + append_started: Arc::new(Mutex::new(Some(append_started_tx))), + release_append: release_append.clone(), + append_finished: append_finished.clone(), + }); + // A limit below the copy's length, so committing the copy schedules an archive transfer. + let service = MultiLayerOplogService::new( + primary.clone(), + nev![ + archive, + Arc::new(BlobOplogArchiveService::new(blob_storage.clone(), 2)) + as Arc + ], + 2, + 1, + ); + let owned_agent_id = OwnedAgentId::new( + EnvironmentId::new(), + &AgentId { + component_id: ComponentId::new(), + agent_id: "fork-target".to_string(), + }, + ); + let target = open_unfenced_fork_target(&service, &owned_agent_id).await; + + let copied = 3; + for _ in 0..copied { + target.add(OplogEntry::no_op(None).rounded()).await.unwrap(); + } + // The transfer's task can run between the fork's final commit and its close; committing + // first and waiting for the transfer to reach the archive makes that certain. + target.commit(CommitLevel::Always).await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), append_started_rx) + .await + .expect("archive transfer did not start") + .expect("archive transfer start signal dropped"); + let handle = Arc::downgrade(&target); + + crate::services::worker_fork::close_fork_target_oplog(target) + .await + .unwrap(); + + assert!( + handle.upgrade().is_none(), + "the archive transfer kept the fork target's handle alive past its close" + ); + let append_completed = append_finished.notified(); + release_append.notify_one(); + assert!( + tokio::time::timeout(Duration::from_millis(100), append_completed) + .await + .is_err(), + "the archive transfer went on after the fork target's handle was closed" + ); + assert_eq!( + primary + .read_source( + &owned_agent_id, + AgentMode::Durable, + OplogIndex::INITIAL, + copied + ) + .await + .len() as u64, + copied, + "a transfer outliving the close dropped the copied entries from the primary oplog" + ); +} + +#[test] +async fn aborting_a_transfer_waits_for_the_prefix_drop_it_handed_to_the_primary( + _tracing: &Tracing, +) { + let (drop_prefix_started_tx, drop_prefix_started_rx) = oneshot::channel(); + let release_drop_prefix = Arc::new(Notify::new()); + let primary_storage = Arc::new(ReadCountingIndexedStorage::blocking_drop_prefix( + drop_prefix_started_tx, + release_drop_prefix.clone(), + )); + let blob_storage = Arc::new(InMemoryBlobStorage::new()); + let primary = Arc::new( + PrimaryOplogService::new( + primary_storage.clone(), + blob_storage.clone(), + 1, + 1, + 100, + RetryConfig::default(), + ) + .await, + ); + let service = MultiLayerOplogService::new( + primary.clone(), + nev![ + Arc::new(CompressedOplogArchiveService::new( + Arc::new(InMemoryIndexedStorage::new()), + 1, + RetryConfig::default(), + )) as Arc, + Arc::new(BlobOplogArchiveService::new(blob_storage.clone(), 2)) + as Arc + ], + 2, + 1, + ); + let owned_agent_id = OwnedAgentId::new( + EnvironmentId::new(), + &AgentId { + component_id: ComponentId::new(), + agent_id: "fork-target-prefix-drop".to_string(), + }, + ); + let target = open_unfenced_fork_target(&service, &owned_agent_id).await; + + for _ in 0..3 { + target.add(OplogEntry::no_op(None).rounded()).await.unwrap(); + } + target.commit(CommitLevel::Always).await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), drop_prefix_started_rx) + .await + .expect("the transfer did not reach the primary's prefix drop") + .expect("prefix drop start signal dropped"); + + // The transfer is waiting for the primary's actor, which is inside the prefix drop. + let abort = tokio::spawn({ + let target = target.clone(); + async move { MultiLayerOplog::try_abort_transfer(&target).await } + }); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + !abort.is_finished(), + "the abort returned while the primary was still dropping the transferred prefix" + ); + + release_drop_prefix.notify_one(); + tokio::time::timeout(Duration::from_secs(1), abort) + .await + .expect("the abort did not return once the prefix drop finished") + .unwrap(); + assert!( + primary + .read_source(&owned_agent_id, AgentMode::Durable, OplogIndex::INITIAL, 3) + .await + .is_empty(), + "the abort returned before the primary finished dropping the transferred prefix" + ); +} diff --git a/golem-worker-executor/src/services/quota.rs b/golem-worker-executor/src/services/quota.rs index 257b4d02e9..1abcbe7b91 100644 --- a/golem-worker-executor/src/services/quota.rs +++ b/golem-worker-executor/src/services/quota.rs @@ -1340,6 +1340,7 @@ mod tests { _port: u16, _pod_name: Option, _executor_id: Uuid, + _previous_shard_epochs: BTreeMap, ) -> Result { unimplemented!() } @@ -1348,6 +1349,7 @@ mod tests { &self, _executor_id: Uuid, _shard_epochs: BTreeMap, + _fenced_shard_epochs: BTreeMap, ) -> Result { unimplemented!() } diff --git a/golem-worker-executor/src/services/rpc.rs b/golem-worker-executor/src/services/rpc.rs index b3bea1a721..0e759f118b 100644 --- a/golem-worker-executor/src/services/rpc.rs +++ b/golem-worker-executor/src/services/rpc.rs @@ -759,11 +759,14 @@ fn rpc_error_from_rejection(rejected: InvocationRejected) -> RpcError { InvocationRejectionReason::NotFound => RpcError::NotFound { details: rejected.error, }, - InvocationRejectionReason::Internal => RpcError::RemoteInternalError { - details: rejected.error, - }, - // The routing miss an executor reports as a typed failure once it has accepted. - InvocationRejectionReason::ShardingNotReady => WorkerExecutorError::ShardingNotReady.into(), + // A routing miss is transient: a retry reaches the shard's owner once the routing table has + // been refreshed. The executor's text names the shard, or for a fenced oplog both epochs, + // so it is kept rather than replaced by a bare "Sharding not ready". + InvocationRejectionReason::Internal | InvocationRejectionReason::ShardingNotReady => { + RpcError::RemoteInternalError { + details: rejected.error, + } + } _ => RpcError::ProtocolError { details: rejected.error, }, @@ -1837,9 +1840,13 @@ impl Rpc for DirectWorkerInvocationRpc { #[cfg(test)] mod protocol_tests { - use super::{RpcError, method_validation_revision, rpc_error_from_failure}; + use super::{ + RpcError, method_validation_revision, rpc_error_from_failure, rpc_error_from_rejection, + }; use crate::services::worker_proxy::WorkerProxyError; - use golem_api_grpc::proto::golem::worker::{InvocationFailure, InvocationFailureKind}; + use golem_api_grpc::proto::golem::worker::{ + InvocationFailure, InvocationFailureKind, InvocationRejected, InvocationRejectionReason, + }; use golem_common::model::agent::{ AgentError as ModelAgentError, InvocationFreshnessDisposition, }; @@ -1917,6 +1924,27 @@ mod protocol_tests { } ); } + + #[test] + fn a_routing_miss_rejection_is_transient_and_keeps_the_executors_detail() { + let detail = + "Oplog write for x fenced: this executor asserted shard epoch 3, the stored epoch is 4"; + let error = rpc_error_from_rejection(InvocationRejected { + reason: InvocationRejectionReason::ShardingNotReady as i32, + error: detail.to_string(), + idempotency_key: None, + agent_id: None, + component_revision: None, + }); + + assert_eq!( + error, + RpcError::RemoteInternalError { + details: detail.to_string(), + } + ); + } + #[test] fn invalid_remote_request_is_an_agent_input_error() { let error = RpcError::from(WorkerExecutorError::invalid_request("wrong argument shape")); diff --git a/golem-worker-executor/src/services/shard.rs b/golem-worker-executor/src/services/shard.rs index 1393c720fb..615c745b43 100644 --- a/golem-worker-executor/src/services/shard.rs +++ b/golem-worker-executor/src/services/shard.rs @@ -14,19 +14,23 @@ use crate::metrics::sharding::*; use crate::model::ShardAssignmentCheck; +use crate::services::oplog::{OplogFence, OplogFenceObserver}; use golem_common::model::{ AgentId, ShardAssignment, ShardDeliveryOutcome, ShardEpoch, ShardId, ShardLeaseRevision, }; use golem_service_base::error::worker_executor::WorkerExecutorError; use itertools::Itertools; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::identity; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::Instant; use tracing::debug; -/// Service for assigning shards to worker executors -pub trait ShardService: Send + Sync { +/// Service for assigning shards to worker executors. +/// +/// Also the oplog fence's observer: the epoch a refused write found belongs to the agent's shard, +/// and this is where the shard an agent routes to is known. +pub trait ShardService: OplogFenceObserver + Send + Sync { /// True once an assignment exists **and** its lease is still live. Gates /// the scheduler's poll loop, which admits work without going through /// `check_admission`. @@ -87,10 +91,25 @@ pub trait ShardService: Send + Sync { fn clear_assignment(&self); fn current_assignment(&self) -> Result; fn try_get_current_assignment(&self) -> Option; + /// The epochs refused oplog writes found on the rows, per shard, that no granted renewal has + /// reported yet. A snapshot: learning more afterwards does not change it. + fn fence_learned_epochs(&self) -> BTreeMap; + /// Retires what a granted renewal reported. An entry goes only while its epoch is still at or + /// below the reported one, so an epoch learned after the snapshot was taken is reported again. + fn retire_fence_learned_epochs(&self, reported: &BTreeMap); } pub struct ShardServiceDefault { shard_assignment: Arc>>, + /// The highest epoch a refused oplog write found on the rows, per shard, until a granted + /// renewal has reported it. + /// + /// Evidence, not ownership: somebody wrote at that epoch, and a shard manager whose state lost + /// history has to mint above it. So it is kept apart from the assignment and survives a lost + /// lease, and it is not filtered by ownership - a fence on a shard this executor no longer + /// holds is still what the manager forgot, and the manager re-mints that shard's owner. At + /// most one entry per shard. Never held across an await. + fence_learned: Mutex>, } impl Default for ShardServiceDefault { @@ -103,6 +122,7 @@ impl ShardServiceDefault { pub fn new() -> Self { Self { shard_assignment: Arc::new(RwLock::new(None)), + fence_learned: Mutex::new(BTreeMap::new()), } } @@ -284,6 +304,60 @@ impl ShardService for ShardServiceDefault { fn try_get_current_assignment(&self) -> Option { self.shard_assignment.read().unwrap().clone() } + + fn fence_learned_epochs(&self) -> BTreeMap { + self.fence_learned.lock().unwrap().clone() + } + + fn retire_fence_learned_epochs(&self, reported: &BTreeMap) { + let mut learned = self.fence_learned.lock().unwrap(); + for (shard_id, reported_epoch) in reported { + if learned + .get(shard_id) + .is_some_and(|learned_epoch| learned_epoch <= reported_epoch) + { + learned.remove(shard_id); + } + } + } +} + +impl OplogFenceObserver for ShardServiceDefault { + fn fenced(&self, fence: &OplogFence) { + // Only a record ahead of the epoch this executor asserted says anything the shard manager + // may have lost. An absent record carries no epoch, and one at or below the assertion is + // not a generation above it. + let Some(stored) = fence + .actual_epoch + .filter(|stored| *stored > fence.expected_epoch) + else { + return; + }; + let shard_id = { + let guard = self.shard_assignment.read().unwrap(); + match guard.as_ref() { + // `ShardId::from_agent_id` divides by the count, and the placeholder a + // registration starts from holds zero. + Some(assignment) if assignment.number_of_shards > 0 => { + ShardId::from_agent_id(&fence.agent_id, assignment.number_of_shards) + } + _ => return, + } + }; + debug!( + agent_id = %fence.agent_id, + %shard_id, + expected_epoch = %fence.expected_epoch, + stored_epoch = %stored, + "Learned a shard epoch from a fenced oplog write" + ); + self.fence_learned + .lock() + .unwrap() + .entry(shard_id) + .and_modify(|learned| *learned = (*learned).max(stored)) + .or_insert(stored); + } } /// Records what every delivery updates: the resulting shard count, and, when the delivery was @@ -375,6 +449,96 @@ mod tests { Instant::now() + Duration::from_secs(60) } + fn fence(agent_id: &AgentId, expected: u64, actual: Option) -> OplogFence { + OplogFence { + agent_id: agent_id.clone(), + expected_epoch: ShardEpoch(expected), + actual_epoch: actual.map(ShardEpoch), + } + } + + fn learned(entries: impl IntoIterator) -> BTreeMap { + entries + .into_iter() + .map(|(shard_id, epoch)| (ShardId::new(shard_id), ShardEpoch(epoch))) + .collect() + } + + /// A refused write's stored epoch is learned under the shard its agent routes to, merged by + /// maximum so a repeated or older report never lowers it. It is evidence rather than + /// ownership: learned whether or not this executor holds the shard, and kept when the lease + /// is lost, because that is exactly when a shard manager that lost history needs it. + #[test] + fn a_fence_learns_the_stored_epoch_keyed_by_the_agents_shard() { + let service = service_holding(&epochs([(3, 0)]), Some(live())); + let on_held = agent_on_shard(3); + + service.fenced(&fence(&on_held, 0, Some(4))); + assert_eq!(service.fence_learned_epochs(), learned([(3, 4)])); + + service.fenced(&fence(&on_held, 0, Some(2))); + assert_eq!( + service.fence_learned_epochs(), + learned([(3, 4)]), + "an older report lowered what was learned" + ); + service.fenced(&fence(&on_held, 0, Some(4))); + assert_eq!( + service.fence_learned_epochs(), + learned([(3, 4)]), + "a repeated report changed what was learned" + ); + + // An absent record carries no epoch, and one at or below the assertion is no generation + // above it. + let on_other = agent_on_shard(1); + service.fenced(&fence(&on_other, 0, None)); + service.fenced(&fence(&on_other, 2, Some(2))); + service.fenced(&fence(&on_other, 3, Some(2))); + assert_eq!(service.fence_learned_epochs(), learned([(3, 4)])); + + let on_unowned = agent_on_shard(5); + service.fenced(&fence(&on_unowned, 0, Some(1))); + assert_eq!(service.fence_learned_epochs(), learned([(3, 4), (5, 1)])); + + service.clear_assignment(); + assert_eq!( + service.fence_learned_epochs(), + learned([(3, 4), (5, 1)]), + "a lost lease dropped the epochs the re-registered executor has to report" + ); + + // With no shard count to route by, a fence is ignored rather than divided by zero. + let unregistered = ShardServiceDefault::new(); + unregistered.fenced(&fence(&on_held, 0, Some(4))); + assert!(unregistered.fence_learned_epochs().is_empty()); + unregistered.with_write_shard_assignment(|shard_assignment| { + *shard_assignment = Some(ShardAssignment::default()) + }); + unregistered.fenced(&fence(&on_held, 0, Some(4))); + assert!(unregistered.fence_learned_epochs().is_empty()); + } + + /// A granted renewal retires the snapshot it reported, and nothing learned since: a higher + /// epoch on a reported shard, or a new shard, still goes with the next renewal. + #[test] + fn retiring_reported_epochs_keeps_ones_learned_since() { + let service = service_holding(&epochs([(3, 0)]), Some(live())); + service.fenced(&fence(&agent_on_shard(3), 0, Some(4))); + let reported = service.fence_learned_epochs(); + + service.fenced(&fence(&agent_on_shard(3), 0, Some(6))); + service.fenced(&fence(&agent_on_shard(5), 0, Some(1))); + service.retire_fence_learned_epochs(&reported); + assert_eq!(service.fence_learned_epochs(), learned([(3, 6), (5, 1)])); + + service.retire_fence_learned_epochs(&learned([(3, 6), (5, 1)])); + assert!(service.fence_learned_epochs().is_empty()); + + service.retire_fence_learned_epochs(&learned([(3, 9)])); + assert!(service.fence_learned_epochs().is_empty()); + } + /// Nothing is installed until a registration: a push, a renewal or a /// revoke that arrives first is refused, never applied to a placeholder /// whose shard count of zero the routing hash would divide by. diff --git a/golem-worker-executor/src/services/shard_manager.rs b/golem-worker-executor/src/services/shard_manager.rs index 67d7b50d0e..8fe323237b 100644 --- a/golem-worker-executor/src/services/shard_manager.rs +++ b/golem-worker-executor/src/services/shard_manager.rs @@ -135,6 +135,13 @@ pub struct GrpcShardManagerService { /// The registration arguments, kept so a re-register after `LeaseNotFound` /// can repeat it without going back through `WorkerExecutorImpl`. registration: RwLock)>>, + /// The shards, with their epochs, this process held before a `LeaseNotFound`, sent with the + /// re-registration that follows so a shard manager whose state lost history mints above them. + /// It has to outlive a failed re-registration: the assignment is already cleared by then, so + /// the next attempt has nothing else to read the set from. Merged by maximum epoch, and emptied + /// once a registration or a renewal succeeds, because the manager then knows this executor + /// again and the next loss carries only what it held since. + carried_claim: RwLock>, /// Weak self-reference, so `register` can spawn the renewal loop without /// the loop keeping this service alive. me: Weak, @@ -196,6 +203,7 @@ impl GrpcShardManagerService { shutdown, executor_id: RwLock::new(Uuid::new_v4()), registration: RwLock::new(None), + carried_claim: RwLock::new(BTreeMap::new()), me: me.clone(), renewal_loop_started: AtomicBool::new(false), retry_backoff: RwLock::new(MIN_RENEWAL_INTERVAL), @@ -393,38 +401,20 @@ impl GrpcShardManagerService { self.record_granted(cadence); cadence } -} - -/// `(expires_at - now) / 3`, floored, so three attempts fit inside one lease. -/// -/// A lease that never expires yields `None`, which parks the -/// renewal loop instead of polling it — there is nothing to renew, and a -/// polling loop would be one wasted RPC per second per executor. -fn renewal_interval_for(expires_at: Option, now: Instant) -> RenewalDelay { - let expires_at = expires_at?; - Some(shard_lease::renewal_interval( - expires_at.saturating_duration_since(now), - )) -} -#[async_trait] -impl ShardManagerService for GrpcShardManagerService { - /// Yes: a real shard manager moves shards between executors, so two of them can believe they - /// own the same agent at once and only the storage can tell them apart. - fn requires_oplog_fencing(&self) -> bool { - true - } - - async fn register( + /// [`ShardManagerService::register`], carrying `previous_claim` to the shard manager: the set + /// this process held under an earlier `executor_id`, or empty on a first registration. + async fn register_with_previous_claim( &self, port: u16, pod_name: Option, + previous_claim: BTreeMap, ) -> Result { *self.registration.write().unwrap() = Some((port, pod_name.clone())); let registration = self .client - .register(port, pod_name, self.executor_id()) + .register(port, pod_name, self.executor_id(), previous_claim) .await?; let number_of_shards: usize = registration.number_of_shards.try_into().map_err(|_| { @@ -462,6 +452,55 @@ impl ShardManagerService for GrpcShardManagerService { Ok(assignment) } + /// Adds the set held right now to the carried claim, keeping the higher epoch where both name + /// a shard, and returns what the next re-registration sends. A shard the current set no longer + /// holds stays in the claim: its epoch is still evidence the manager may have lost. + fn carry_current_claim(&self) -> BTreeMap { + let held = self + .shard_service + .try_get_current_assignment() + .map(|assignment| assignment.claim()) + .unwrap_or_default(); + let mut carried = self.carried_claim.write().unwrap(); + for (shard_id, epoch) in held { + carried + .entry(shard_id) + .and_modify(|carried_epoch| *carried_epoch = (*carried_epoch).max(epoch)) + .or_insert(epoch); + } + carried.clone() + } +} + +/// `(expires_at - now) / 3`, floored, so three attempts fit inside one lease. +/// +/// A lease that never expires yields `None`, which parks the +/// renewal loop instead of polling it — there is nothing to renew, and a +/// polling loop would be one wasted RPC per second per executor. +fn renewal_interval_for(expires_at: Option, now: Instant) -> RenewalDelay { + let expires_at = expires_at?; + Some(shard_lease::renewal_interval( + expires_at.saturating_duration_since(now), + )) +} + +#[async_trait] +impl ShardManagerService for GrpcShardManagerService { + /// Yes: a real shard manager moves shards between executors, so two of them can believe they + /// own the same agent at once and only the storage can tell them apart. + fn requires_oplog_fencing(&self) -> bool { + true + } + + async fn register( + &self, + port: u16, + pod_name: Option, + ) -> Result { + self.register_with_previous_claim(port, pod_name, BTreeMap::new()) + .await + } + async fn renew_shard_lease(&self) -> RenewalDelay { let claim = match self.shard_service.current_assignment() { Ok(assignment) => assignment.claim(), @@ -472,22 +511,42 @@ impl ShardManagerService for GrpcShardManagerService { }; let executor_id = self.executor_id(); + // Reported on every renewal and retired only by a grant, so a renewal that is refused or + // lost leaves them for the next one; a manager that already applied them moves nothing + // the second time. + let fenced = self.shard_service.fence_learned_epochs(); + if !fenced.is_empty() { + info!( + fenced_shard_epochs = ?fenced, + "Reporting shard epochs learned from fenced oplog writes with the lease renewal" + ); + } let deadline = self.rpc_deadline(); - let renewed = - match tokio::time::timeout(deadline, self.client.renew_shard_lease(executor_id, claim)) - .await - { - Ok(renewed) => renewed, - Err(_elapsed) => { - warn!( - deadline_ms = deadline.as_millis(), - "Shard lease renewal did not answer in time; backing off" - ); - return self.next_retry_delay(); - } - }; + let renewed = match tokio::time::timeout( + deadline, + self.client + .renew_shard_lease(executor_id, claim, fenced.clone()), + ) + .await + { + Ok(renewed) => renewed, + Err(_elapsed) => { + warn!( + deadline_ms = deadline.as_millis(), + "Shard lease renewal did not answer in time; backing off" + ); + return self.next_retry_delay(); + } + }; match renewed { Ok(lease) => { + // The manager knows this executor, so its state has the history a carried claim + // was kept to restore; one stored by a re-registration whose reply was lost + // lands here too. + self.carried_claim.write().unwrap().clear(); + // Stored with this renewal. Only what was sent is retired: an epoch learned while + // the renewal was in flight goes with the next one. + self.shard_service.retire_fence_learned_epochs(&fenced); self.adopt_lease(lease.shard_epochs, lease.expires_at, lease.revision) .await } @@ -495,11 +554,19 @@ impl ShardManagerService for GrpcShardManagerService { // The manager no longer knows this executor. Drop every shard // (an empty set fences every agent) and come back as a new // instance at the same address, which is the restarted-executor - // path the manager already handles. + // path the manager already handles. The set held until now goes + // with the registration: if the manager's state was wiped or + // replaced, it has lost the epochs this executor's oplog rows + // were written at, and would otherwise mint below them. Epochs + // learned from fenced writes do not go with it: a registration + // applies what it carries as this executor's own claim, and + // would stamp another writer's epoch onto its entries. They + // ride on the first renewal under the fresh id instead. warn!( details, "Shard lease not found, clearing the assignment and re-registering" ); + let previous_claim = self.carry_current_claim(); self.shard_service.clear_assignment(); let fresh_executor_id = Uuid::new_v4(); *self.executor_id.write().unwrap() = fresh_executor_id; @@ -510,8 +577,12 @@ impl ShardManagerService for GrpcShardManagerService { error!("Cannot re-register: this executor never completed a registration"); self.next_retry_delay() } - Some((port, pod_name)) => match self.register(port, pod_name).await { + Some((port, pod_name)) => match self + .register_with_previous_claim(port, pod_name, previous_claim) + .await + { Ok(assignment) => { + self.carried_claim.write().unwrap().clear(); self.shard_service.register( assignment.number_of_shards, &assignment.shard_epochs, @@ -627,6 +698,7 @@ impl ShardManagerService for ShardManagerServiceSingleShard { #[cfg(test)] mod tests { use super::*; + use crate::services::oplog::{OplogFence, OplogFenceObserver}; use crate::services::shard::ShardServiceDefault; use golem_common::model::component::ComponentId; use golem_common::model::environment::EnvironmentId; @@ -710,8 +782,11 @@ mod tests { renew_gate: StdMutex>>, /// The same for a deregistration. deregister_gate: StdMutex>>, - register_calls: StdMutex>, + /// Each registration's executor id and the previous claim it carried. + register_calls: StdMutex)>>, renew_calls: StdMutex)>>, + /// The fenced epochs each renewal reported, in the order of `renew_calls`. + renew_fenced_calls: StdMutex>>, deregister_calls: StdMutex)>>, } @@ -724,6 +799,7 @@ mod tests { deregister_gate: StdMutex::new(None), register_calls: StdMutex::new(Vec::new()), renew_calls: StdMutex::new(Vec::new()), + renew_fenced_calls: StdMutex::new(Vec::new()), deregister_calls: StdMutex::new(Vec::new()), } } @@ -757,7 +833,7 @@ mod tests { self } - fn register_calls(&self) -> Vec { + fn register_calls(&self) -> Vec<(Uuid, BTreeMap)> { self.register_calls.lock().unwrap().clone() } @@ -765,6 +841,10 @@ mod tests { self.renew_calls.lock().unwrap().clone() } + fn renew_fenced_calls(&self) -> Vec> { + self.renew_fenced_calls.lock().unwrap().clone() + } + fn deregister_calls(&self) -> Vec<(Uuid, BTreeMap)> { self.deregister_calls.lock().unwrap().clone() } @@ -781,8 +861,12 @@ mod tests { _port: u16, _pod_name: Option, executor_id: Uuid, + previous_shard_epochs: BTreeMap, ) -> Result { - self.register_calls.lock().unwrap().push(executor_id); + self.register_calls + .lock() + .unwrap() + .push((executor_id, previous_shard_epochs)); let guard = self.register_fn.lock().unwrap(); let f = guard.as_ref().expect("register_fn not configured"); f(executor_id) @@ -792,11 +876,16 @@ mod tests { &self, executor_id: Uuid, shard_epochs: BTreeMap, + fenced_shard_epochs: BTreeMap, ) -> Result { self.renew_calls .lock() .unwrap() .push((executor_id, shard_epochs.clone())); + self.renew_fenced_calls + .lock() + .unwrap() + .push(fenced_shard_epochs); // Cloned out before the await: the guard must not be held across it. let gate = self.renew_gate.lock().unwrap().clone(); if let Some(gate) = gate { @@ -1530,14 +1619,14 @@ mod tests { assignment.expires_at, assignment.revision, ); - let original_executor_id = mock.register_calls()[0]; + let original_executor_id = mock.register_calls()[0].0; service.renew_shard_lease().await; let register_calls = mock.register_calls(); assert_eq!(register_calls.len(), 2, "a lost lease must re-register"); assert_ne!( - register_calls[1], original_executor_id, + register_calls[1].0, original_executor_id, "the re-registration must come back as a new instance, under a fresh UUID" ); assert!( @@ -1549,6 +1638,267 @@ mod tests { assert_eq!(assignment.expires_at, Some(fresh_expiry)); } + /// A lost lease re-registers carrying the set it held, so a shard manager whose state was wiped + /// mints above the epochs this executor's oplog rows were written at. The set has to survive a + /// failed re-registration, which already cleared the assignment it was read from; a set that + /// arrives afterwards naming one of its shards lower must not lower it; and once a registration + /// succeeds the next loss carries only what that registration granted. + #[test] + async fn a_lost_lease_re_registers_carrying_the_shards_it_held() { + let expiry = Instant::now() + Duration::from_secs(120); + let attempts = Arc::new(AtomicUsize::new(0)); + let attempt = attempts.clone(); + let mock = Arc::new( + MockShardManager::new() + .with_register(move |_| match attempt.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(registration(expiry, [(2, 5)])), + 1 | 2 => Err(ShardManagerError::InternalServerError( + "shard manager down".to_string(), + )), + _ => Ok(registration(expiry, [(3, 1)])), + }) + .with_renew(|_, _| Err(ShardLeaseError::LeaseNotFound("unknown".to_string()))), + ); + let (service, shard_service) = make_service(mock.clone(), Shutdown::new()); + + let assignment = service.register(PORT, None).await.unwrap(); + shard_service.register( + assignment.number_of_shards, + &assignment.shard_epochs, + assignment.expires_at, + assignment.revision, + ); + + service.renew_shard_lease().await; + let register_calls = mock.register_calls(); + assert_eq!(register_calls.len(), 2); + assert_eq!( + register_calls[0].1, + BTreeMap::new(), + "a first registration has nothing to carry" + ); + assert_eq!(register_calls[1].1, claim([(2, 5)])); + + // The failed attempt left the assignment cleared, so this pass renews an empty claim. + service.renew_shard_lease().await; + assert_eq!(mock.renew_calls()[1].1, BTreeMap::new()); + let register_calls = mock.register_calls(); + assert_eq!(register_calls.len(), 3); + assert_eq!( + register_calls[2].1, + claim([(2, 5)]), + "the set held before the loss must outlive the failed re-registration" + ); + + // A delivery naming shard 2 below the carried epoch: the claim keeps the higher one, + // because the oplog rows were written at it whatever the newer set says. + shard_service.register( + SHARDS, + &epochs([(2, 3), (4, 1)]), + Some(expiry), + ShardLeaseRevision(1), + ); + service.renew_shard_lease().await; + let register_calls = mock.register_calls(); + assert_eq!(register_calls.len(), 4); + assert_eq!(register_calls[3].1, claim([(2, 5), (4, 1)])); + assert_eq!( + shard_service.current_assignment().unwrap().shard_epochs, + epochs([(3, 1)]) + ); + + service.renew_shard_lease().await; + let register_calls = mock.register_calls(); + assert_eq!(register_calls.len(), 5); + assert_eq!( + register_calls[4].1, + claim([(3, 1)]), + "a successful registration retires the claim it carried" + ); + } + + /// A re-registration can be stored by the manager and still fail here, when its reply is lost. + /// The next renewal under the fresh id is then granted, which is the manager saying it knows + /// this executor again: the carried claim is dropped there, so the next loss does not send + /// epochs from before the last one. + #[test] + async fn a_renewal_granted_after_a_failed_re_registration_drops_the_carried_claim() { + let expiry = Instant::now() + Duration::from_secs(120); + let registrations = Arc::new(AtomicUsize::new(0)); + let registration_attempt = registrations.clone(); + let renewals = Arc::new(AtomicUsize::new(0)); + let renewal_attempt = renewals.clone(); + let mock = Arc::new( + MockShardManager::new() + .with_register( + move |_| match registration_attempt.fetch_add(1, Ordering::SeqCst) { + 0 => Ok(registration(expiry, [(2, 5)])), + 1 => Err(ShardManagerError::InternalServerError( + "reply lost".to_string(), + )), + _ => Ok(registration(expiry, [(3, 1)])), + }, + ) + .with_renew( + move |_, _| match renewal_attempt.fetch_add(1, Ordering::SeqCst) { + 1 => Ok(ShardLease { + shard_epochs: claim([(3, 1)]), + expires_at: expiry, + revision: ShardLeaseRevision(1), + }), + _ => Err(ShardLeaseError::LeaseNotFound("unknown".to_string())), + }, + ), + ); + let (service, shard_service) = make_service(mock.clone(), Shutdown::new()); + + let assignment = service.register(PORT, None).await.unwrap(); + shard_service.register( + assignment.number_of_shards, + &assignment.shard_epochs, + assignment.expires_at, + assignment.revision, + ); + + service.renew_shard_lease().await; + assert_eq!(mock.register_calls()[1].1, claim([(2, 5)])); + + service.renew_shard_lease().await; + assert_eq!( + shard_service.current_assignment().unwrap().shard_epochs, + epochs([(3, 1)]) + ); + + service.renew_shard_lease().await; + let register_calls = mock.register_calls(); + assert_eq!(register_calls.len(), 3); + assert_eq!( + register_calls[2].1, + claim([(3, 1)]), + "the granted renewal retired the claim carried from the earlier loss" + ); + } + + /// The epoch a fenced write found, on the shard `agent_on_shard(shard)` routes to. + fn fence_on_shard(shard: i64, expected: u64, stored: u64) -> OplogFence { + OplogFence { + agent_id: agent_on_shard(shard), + expected_epoch: ShardEpoch(expected), + actual_epoch: Some(ShardEpoch(stored)), + } + } + + /// Epochs learned from fenced oplog writes ride on the next renewal beside the claim, so a + /// shard manager whose state lost history can mint above them. A granted renewal has stored + /// whatever they moved, so it retires them and the next renewal reports nothing. + #[test] + async fn a_renewal_reports_fence_learned_epochs_and_retires_them_once_granted() { + let expiry = Instant::now() + Duration::from_secs(120); + let mock = Arc::new( + MockShardManager::new() + .with_register(move |_| Ok(registration(expiry, [(2, 5)]))) + .with_renew(move |_, claimed| { + Ok(ShardLease { + shard_epochs: claimed, + expires_at: expiry, + revision: ShardLeaseRevision(1), + }) + }), + ); + let (service, shard_service) = make_service(mock.clone(), Shutdown::new()); + + let assignment = service.register(PORT, None).await.unwrap(); + shard_service.register( + assignment.number_of_shards, + &assignment.shard_epochs, + assignment.expires_at, + assignment.revision, + ); + shard_service.fenced(&fence_on_shard(2, 5, 7)); + + service.renew_shard_lease().await; + assert_eq!(mock.renew_calls()[0].1, claim([(2, 5)])); + assert_eq!(mock.renew_fenced_calls()[0], claim([(2, 7)])); + assert_eq!( + shard_service.fence_learned_epochs(), + BTreeMap::new(), + "a granted renewal retires the epochs it reported" + ); + + service.renew_shard_lease().await; + assert_eq!( + mock.renew_fenced_calls()[1], + BTreeMap::new(), + "an epoch the manager already stored was reported again" + ); + } + + /// Nothing but a grant says the manager stored a report. A renewal lost in transport, and one + /// refused as a lease not found, keep the learned epochs for the next pass. The + /// re-registration after the refusal carries only the held claim, because a registration + /// applies what it carries as this executor's own; the learned epochs go on the first renewal + /// under the fresh id. + #[test] + async fn a_refused_renewal_keeps_fence_learned_epochs_for_the_next_one() { + let expiry = Instant::now() + Duration::from_secs(120); + let renewals = Arc::new(AtomicUsize::new(0)); + let renewal_attempt = renewals.clone(); + let mock = Arc::new( + MockShardManager::new() + .with_register(move |_| Ok(registration(expiry, [(2, 5)]))) + .with_renew(move |_, claimed| { + match renewal_attempt.fetch_add(1, Ordering::SeqCst) { + 0 => Err(ShardLeaseError::InternalServerError( + "shard manager down".to_string(), + )), + 1 => Err(ShardLeaseError::LeaseNotFound("unknown".to_string())), + _ => Ok(ShardLease { + shard_epochs: claimed, + expires_at: expiry, + revision: ShardLeaseRevision(1), + }), + } + }), + ); + let (service, shard_service) = make_service(mock.clone(), Shutdown::new()); + + let assignment = service.register(PORT, None).await.unwrap(); + shard_service.register( + assignment.number_of_shards, + &assignment.shard_epochs, + assignment.expires_at, + assignment.revision, + ); + shard_service.fenced(&fence_on_shard(2, 5, 7)); + + service.renew_shard_lease().await; + assert_eq!(mock.renew_fenced_calls()[0], claim([(2, 7)])); + assert_eq!( + shard_service.fence_learned_epochs(), + claim([(2, 7)]), + "a renewal lost in transport retired what it never delivered" + ); + + service.renew_shard_lease().await; + assert_eq!(mock.renew_fenced_calls()[1], claim([(2, 7)])); + let register_calls = mock.register_calls(); + assert_eq!(register_calls.len(), 2, "a lost lease must re-register"); + assert_eq!( + register_calls[1].1, + claim([(2, 5)]), + "the re-registration carries the held claim, never a fenced epoch" + ); + assert_eq!( + shard_service.fence_learned_epochs(), + claim([(2, 7)]), + "a refused renewal retired what the manager never stored" + ); + + service.renew_shard_lease().await; + assert_eq!(mock.renew_fenced_calls()[2], claim([(2, 7)])); + assert_eq!(shard_service.fence_learned_epochs(), BTreeMap::new()); + } + /// A renewal that answers with shards this executor did not claim is the /// shard manager correcting a push that never arrived, so it has to recover /// agents for them exactly as a push would. The common path — the same set diff --git a/golem-worker-executor/src/services/worker_fork.rs b/golem-worker-executor/src/services/worker_fork.rs index 737e1f4ca3..f2d00d8caf 100644 --- a/golem-worker-executor/src/services/worker_fork.rs +++ b/golem-worker-executor/src/services/worker_fork.rs @@ -21,7 +21,7 @@ use crate::metrics::workers::record_worker_call; use crate::model::ExecutionStatus; use crate::services::events::Events; use crate::services::oplog::plugin::OplogProcessorPlugin; -use crate::services::oplog::{CommitLevel, Oplog, OplogOps}; +use crate::services::oplog::{CommitLevel, MultiLayerOplog, Oplog, OplogOps}; use crate::services::resource_limits::ResourceLimits; use crate::services::rpc::Rpc; use crate::services::shard::ShardService; @@ -624,8 +624,11 @@ impl DefaultWorkerFork { }, ))), // Unfenced: the target's shard may belong to another executor, and - // this is a one-shot copy, not a live oplog. Its owner writes the metadata - // row on its first open. + // this is a one-shot copy, not a live oplog. The handle is closed, with + // any archive transfer it scheduled ended, before the target is resumed + // (`close_fork_target_oplog`), and the open-oplog cache never hands a + // handle opened without an epoch to an opener that asserts one, so the + // owner's first open builds its own handle and writes the metadata row. None, ) .await; @@ -784,13 +787,19 @@ fn rewrite_forked_oplog_entry( ) -> OplogEntry { match &mut entry { OplogEntry::AgentInvocationStarted { - wallet_pin: Some(wallet_pin), + wallet_pin, + shard_epoch, .. } => { - wallet_pin.wallet_token.wallet_id_hash = CardHolder::Agent(AgentCardHolder { - agent_id: target_agent_id.clone(), - }) - .wallet_id_hash(); + // The source's epoch names a generation of the source's shard. The copy lands in an + // oplog opened without an epoch to assert, which the field records as `None`. + *shard_epoch = None; + if let Some(wallet_pin) = wallet_pin { + wallet_pin.wallet_token.wallet_id_hash = CardHolder::Agent(AgentCardHolder { + agent_id: target_agent_id.clone(), + }) + .wallet_id_hash(); + } } OplogEntry::CardEventQueued { event: QueuedCardEvent::TransferStarted(event), @@ -824,6 +833,23 @@ fn rewrite_forked_oplog_entry( entry } +/// Commits the fork target's copied oplog and closes its handle, before the target is resumed. +/// +/// The handle asserts no epoch, so nothing it started may still be writing once the target's +/// owner opens the oplog at its own epoch. Dropping it does not ensure that: when the copy reaches +/// the entry count limit, the final commit schedules an archive transfer that holds its own +/// reference to the handle and ends by dropping the primary's prefix, and deleting the primary +/// oplog once that empties it, over whatever the owner appended meanwhile. The transfer is ended +/// through the handle, not the agent-keyed transfer registry, which the owner's open overwrites. +pub(crate) async fn close_fork_target_oplog( + new_oplog: Arc, +) -> Result<(), WorkerExecutorError> { + new_oplog.commit(CommitLevel::Always).await?; + MultiLayerOplog::try_abort_transfer(&new_oplog).await; + drop(new_oplog); + Ok(()) +} + #[async_trait] impl WorkerForkService for DefaultWorkerFork { async fn fork( @@ -843,7 +869,7 @@ impl WorkerForkService for DefaultWorkerFork { ) .await?; - new_oplog.commit(CommitLevel::Always).await?; + close_fork_target_oplog(new_oplog).await?; // We go through worker proxy to resume the worker // as we need to make sure as it may live in another worker executor, @@ -933,7 +959,7 @@ impl WorkerForkService for DefaultWorkerFork { .await?; } - new_oplog.commit(CommitLevel::Always).await?; + close_fork_target_oplog(new_oplog).await?; // We go through worker proxy to resume the worker // as we need to make sure as it may live in another worker executor, @@ -995,21 +1021,56 @@ mod tests { pinned_card_ids: Vec::new(), scope_card_id: None, }), - shard_epoch: None, + shard_epoch: Some(7), }; match rewrite_forked_oplog_entry(entry, &source, &target) { OplogEntry::AgentInvocationStarted { wallet_pin: Some(wallet_pin), + shard_epoch, .. - } => assert_eq!( - wallet_pin.wallet_token.wallet_id_hash, - CardHolder::Agent(AgentCardHolder { agent_id: target }).wallet_id_hash() - ), + } => { + assert_eq!( + wallet_pin.wallet_token.wallet_id_hash, + CardHolder::Agent(AgentCardHolder { agent_id: target }).wallet_id_hash() + ); + assert_eq!(shard_epoch, None); + } other => panic!("expected pinned invocation start, got {other:?}"), } } + #[test] + fn fork_clears_the_source_shard_epoch_from_copied_invocations() { + let source = agent_id("source"); + let target = agent_id("target"); + let entry = OplogEntry::AgentInvocationStarted { + timestamp: Timestamp::now_utc(), + idempotency_key: IdempotencyKey::new("fork-shard-epoch".to_string()), + payload: OplogPayload::Inline(Box::new(AgentInvocationPayload::AgentMethod { + method_name: "test".to_string(), + input: SchemaValue::Record { fields: Vec::new() }, + principal: Principal::anonymous(), + scope_card: None, + })), + trace_id: TraceId::generate(), + trace_states: Vec::new(), + invocation_context: Vec::new(), + wallet_pin: None, + shard_epoch: Some(7), + }; + + // The source's epoch belongs to the source's shard; the target's copy asserts none. + match rewrite_forked_oplog_entry(entry, &source, &target) { + OplogEntry::AgentInvocationStarted { + wallet_pin: None, + shard_epoch, + .. + } => assert_eq!(shard_epoch, None), + other => panic!("expected unpinned invocation start, got {other:?}"), + } + } + #[test] fn fork_rewrites_only_local_transfer_holders() { let source = agent_id("source"); diff --git a/golem-worker-executor/src/storage/indexed/mod.rs b/golem-worker-executor/src/storage/indexed/mod.rs index ccf1522a58..911ba2fa3e 100644 --- a/golem-worker-executor/src/storage/indexed/mod.rs +++ b/golem-worker-executor/src/storage/indexed/mod.rs @@ -319,7 +319,11 @@ pub trait IndexedStorage: Debug + Sync { /// key has none. /// /// Monotonic rather than a plain overwrite so that a writer holding a stale epoch cannot walk - /// the record backwards and un-fence itself against the current owner. + /// the record backwards and un-fence itself against the current owner. That holds only for a + /// key that already has a record. A key with none accepts any epoch, whether it was never + /// written, removed by [`Self::delete_oplog_metadata`], or written before the record existed. + /// For such a key the fence cannot tell a stale executor's first open from the owner's; only + /// the lease's admission check bounds that window. /// /// The default does nothing and accepts everything: a backend that cannot fence has no record /// to keep. @@ -338,6 +342,9 @@ pub trait IndexedStorage: Debug + Sync { /// before its entries are, so that a writer still holding the old epoch is fenced by the /// absent record rather than appending to an oplog that is being removed. /// + /// Removing the record also forgets its epoch. A writer that already has the oplog open is + /// fenced by the absent record, but a later open at any epoch writes a new one. + /// /// Idempotent. The default does nothing. async fn delete_oplog_metadata( &self, diff --git a/golem-worker-executor/src/storage/indexed/postgres.rs b/golem-worker-executor/src/storage/indexed/postgres.rs index 091022cab8..97e89a9893 100644 --- a/golem-worker-executor/src/storage/indexed/postgres.rs +++ b/golem-worker-executor/src/storage/indexed/postgres.rs @@ -280,9 +280,10 @@ impl IndexedStorage for PostgresIndexedStorage { Ok((new_cursor, keys)) } - /// Delegates to [`Self::append_many`] so there is exactly one fenced write path: the epoch - /// check has to happen in the same transaction as the insert, and a lone `INSERT` is not in - /// one. The permit is acquired there, not here. + /// Delegates to [`Self::append_many`] so a single entry and a batch share the id validation, + /// the permit and the epoch check. An entry that asserts an epoch is checked in the same + /// transaction as its insert, like a batch; one that asserts nothing is a single autocommit + /// `INSERT`. The permit is acquired there, not here. async fn append( &self, svc_name: &'static str, @@ -328,6 +329,27 @@ impl IndexedStorage for PostgresIndexedStorage { Self::to_i64(*id, "id")?; } + // With no epoch asserted there is nothing to check atomically with the insert, so a lone + // entry is one autocommit `INSERT` rather than a transaction held open around it. An + // entry that asserts an epoch takes the transaction below, as a batch does. + if let (None, [(id, value)]) = (shard_epoch, pairs.as_ref()) { + return self + .pool + .with_rw(svc_name, api_name) + .execute( + sqlx::query( + "INSERT INTO index_storage (namespace, key, id, value) VALUES ($1, $2, $3, $4);", + ) + .bind(namespace) + .bind(key) + .bind(i64::try_from(*id).expect("validated oplog index")) + .bind(value.as_ref()), + ) + .await + .map(|_| ()) + .map_err(|err| Self::classify_repo_error(err, primary_oplog_insert)); + } + self.pool .with_tx_err::<(), FencedTxError, _>(svc_name, api_name, |tx| { async move { @@ -394,9 +416,10 @@ impl IndexedStorage for PostgresIndexedStorage { /// Monotonic compare-and-set on the epoch authorised to write this key. /// /// The `WHERE` on the conflict path is what makes it monotonic: a lower epoch updates no row, - /// so a writer holding a stale epoch cannot walk the record back and un-fence itself against - /// the current owner. Postgres reports one row affected for an insert and for an accepted - /// update, and zero when the `WHERE` excludes it. + /// so while a record exists a writer holding a stale epoch cannot walk it back and un-fence + /// itself against the current owner. With no record there is no conflict and any epoch is + /// inserted. Postgres reports one row affected for an insert and for an accepted update, and + /// zero when the `WHERE` excludes it. async fn upsert_oplog_metadata( &self, svc_name: &'static str, diff --git a/golem-worker-executor/src/storage/indexed/sqlite.rs b/golem-worker-executor/src/storage/indexed/sqlite.rs index 1fcf4f145e..24dac0ec42 100644 --- a/golem-worker-executor/src/storage/indexed/sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/sqlite.rs @@ -334,8 +334,9 @@ impl IndexedStorage for SqliteIndexedStorage { } /// Monotonic compare-and-set: the `WHERE` on the conflict path means a lower epoch updates no - /// row, so a stale writer cannot walk the record back and un-fence itself. The unqualified - /// `epoch` there is the existing row's. + /// row, so while a record exists a stale writer cannot walk it back and un-fence itself. With + /// no record there is no conflict and any epoch is inserted. The unqualified `epoch` there is + /// the existing row's. async fn upsert_oplog_metadata( &self, svc_name: &'static str, diff --git a/golem-worker-executor/src/worker/invocation_loop.rs b/golem-worker-executor/src/worker/invocation_loop.rs index 3123337515..b2be0f136d 100644 --- a/golem-worker-executor/src/worker/invocation_loop.rs +++ b/golem-worker-executor/src/worker/invocation_loop.rs @@ -30,7 +30,7 @@ use crate::worker::invocation::{ use crate::worker::status_checkpointer; use crate::worker::{ CreateWorkerInstanceError, FinalWorkerState, PendingLiveInvocationDisposition, - PendingWorkerInterrupt, QueuedWorkerInvocation, RetryDecision, RunningAgent, + PendingWorkerInterrupt, QueuedWorkerInvocation, RelinquishReason, RetryDecision, RunningAgent, RunningAgentRuntime, RunningWorker, UnloadReason, UnloadRequest, Worker, WorkerCommand, WorkerInterruptState, WorkerRunningAgent, WorkerTrace, }; @@ -316,6 +316,14 @@ impl InvocationLoop { InterruptKind::ShardLost => { // Nothing is written: the oplog belongs to the shard's new owner // now. Whoever was waiting for this start is told to look there. + // + // A fence found by a host call during instantiation arrives here + // without `relinquish()` having run, so the agent is marked given up + // now: the stop below then tears its entity bodies down as + // `ShardLost`, fails its waiters and removes only this generation. A + // reason `relinquish()` already recorded is kept. + self.parent + .mark_relinquished(RelinquishReason::Fenced(None)); self.parent.complete_startup( self.start_attempt, Err(WorkerExecutorError::ShardingNotReady), @@ -565,18 +573,11 @@ impl InvocationLoop { .try_get_active_agent(&self.owned_agent_id) .await { - let owner_failure = final_interrupt - .map(OwnerFailureWinner::Lifecycle) - .or_else(|| { - recovery_failure - .clone() - .map(OwnerFailureWinner::Infrastructure) - }) - .unwrap_or_else(|| { - OwnerFailureWinner::Lifecycle( - InterruptKind::Interrupt(Timestamp::now_utc()), - ) - }); + let owner_failure = exit_owner_failure( + self.parent.relinquished_owner_failure(), + final_interrupt, + recovery_failure.as_ref(), + ); active_agent.fence_entity_bodies(owner_failure).await; } if let Some(error) = Self::unload_running_agent( @@ -802,9 +803,10 @@ impl InvocationLoop { .try_get_active_agent(&self.owned_agent_id) .await { - let failure = startup_failure.clone().map_or_else( - || OwnerFailureWinner::Lifecycle(InterruptKind::Interrupt(Timestamp::now_utc())), - OwnerFailureWinner::Infrastructure, + let failure = exit_owner_failure( + self.parent.relinquished_owner_failure(), + None, + startup_failure.as_ref(), ); active_agent.fence_entity_bodies(failure).await; } @@ -2368,6 +2370,23 @@ impl Invocation<'_, Ctx> { .and_then(|result| result) { tracing::error!(%error, "Failed to complete durable streaming session"); + if let Some(reason) = + session_completion_relinquishment(&error, self.parent.oplog.fence()) + { + self.parent.mark_relinquished(reason); + let decision = self + .store + .data_mut() + .on_invocation_failure( + &full_function_name, + &TrapType::Interrupt(InterruptKind::ShardLost), + ) + .await; + return failed_agent_invocation_outcome( + self.parent.agent_mode(), + decision, + ); + } return failed_agent_invocation_outcome( self.parent.agent_mode(), RetryDecision::Immediate, @@ -2413,18 +2432,23 @@ impl Invocation<'_, Ctx> { failed_agent_invocation_outcome(self.parent.agent_mode(), decision) } Err(error) => { + // The success hook commits `AgentInvocationFinished`; if the storage refused that + // commit the oplog has latched the fence, and the failure is a lost shard rather + // than an internal error. + let trap_type = self + .store + .data() + .durable_ctx() + .trap_type_under_latched_fence(TrapType::Error { + error: AgentError::InternalError(error.to_string()), + retry_from: OplogIndex::INITIAL, + in_atomic_region: false, + atomic_region_had_side_effects: false, + semantic_trap_retry_override: None, + }); self.store .data_mut() - .on_invocation_failure( - &full_function_name, - &TrapType::Error { - error: AgentError::InternalError(error.to_string()), - retry_from: OplogIndex::INITIAL, - in_atomic_region: false, - atomic_region_had_side_effects: false, - semantic_trap_retry_override: None, - }, - ) + .on_invocation_failure(&full_function_name, &trap_type) .await; if self.uses_streams { let _ = self @@ -2482,6 +2506,14 @@ impl Invocation<'_, Ctx> { )), }, }; + // A fence that reached the guest through a `String` boundary classifies as an ordinary + // failure; the latch still says the oplog is finished, so the agent is given up, not retried. + let trap_type = trap_type.map(|trap_type| { + self.store + .data() + .durable_ctx() + .trap_type_under_latched_fence(trap_type) + }); let decision = match trap_type { Some(trap_type) => { self.store @@ -2987,6 +3019,30 @@ fn successful_agent_invocation_outcome( } } +/// The failure a loop's exit tears the agent's entity bodies down with. +/// +/// A relinquished agent's shard moved, and that wins over any lifecycle interrupt still queued and +/// over a recovery failure: the bodies must not report an API interrupt or a fault for an agent +/// that simply has a new owner. A relinquishment first discovered by the stop's own commit, which +/// runs after this choice, cannot be reflected, because by then the bodies are already torn down; +/// the fence still holds, and that stop still fails the waiters and removes the generation. +fn exit_owner_failure( + relinquished: Option, + final_interrupt: Option, + recovery_failure: Option<&WorkerExecutorError>, +) -> OwnerFailureWinner { + relinquished + .or_else(|| final_interrupt.map(OwnerFailureWinner::Lifecycle)) + .or_else(|| { + recovery_failure + .cloned() + .map(OwnerFailureWinner::Infrastructure) + }) + .unwrap_or_else(|| { + OwnerFailureWinner::Lifecycle(InterruptKind::Interrupt(Timestamp::now_utc())) + }) +} + fn failed_agent_invocation_outcome( agent_mode: AgentMode, decision: RetryDecision, @@ -2998,6 +3054,24 @@ fn failed_agent_invocation_outcome( } } +/// Why the agent is given up when its streaming session could not be completed, if it is. +/// +/// A commit the storage refused reaches the completion as `OplogFenced` or flattened into a +/// runtime error, and in both cases the oplog has latched the fence. An in-place retry would +/// reopen the oplog at the epoch that was just refused, so the agent is relinquished instead. +fn session_completion_relinquishment( + error: &WorkerExecutorError, + latched: Option, +) -> Option { + match latched { + Some(fence) => Some(RelinquishReason::Fenced(Some(Box::new(fence)))), + None if matches!(error, WorkerExecutorError::OplogFenced { .. }) => { + Some(RelinquishReason::Fenced(None)) + } + None => None, + } +} + fn should_cleanup_terminal_ephemeral_invocation( agent_mode: AgentMode, is_agent_component: bool, @@ -3057,13 +3131,14 @@ mod tests { use super::{ CommandOutcome, ConcurrentAgentPermitState, InvocationLoop, PeriodicSnapshotAction, ResidentAgentOwnership, ResidentWakeup, catch_invocation_loop_panic, - close_usage_before_delete, coalesce_filesystem_limit_update, + close_usage_before_delete, coalesce_filesystem_limit_update, exit_owner_failure, failed_agent_invocation_outcome, finish_filesystem_limit_unload, periodic_snapshot_failure_outcome, publish_unload_outcome, run_invocation_loop_task, - snapshot_action_at, snapshot_baseline_timestamp, spawn_module_owned_unload, - successful_agent_invocation_outcome, unload_resident_agent_ownership, - wait_for_resident_wakeup, + session_completion_relinquishment, snapshot_action_at, snapshot_baseline_timestamp, + spawn_module_owned_unload, successful_agent_invocation_outcome, + unload_resident_agent_ownership, wait_for_resident_wakeup, }; + use crate::durable_host::tool::operation::OwnerFailureWinner; use crate::sandbox_filesystem::ScriptedSandboxFilesystem; use crate::services::active_agents::stop_loaded_idle_if_eligible; use crate::services::agent_filesystem::{ @@ -3076,15 +3151,15 @@ mod tests { use crate::worker::invocation::InvokeResult; use crate::worker::{ EvictionClass, FilesystemPressureEligibility, FinalWorkerState, - PendingLiveInvocationDisposition, RetryDecision, RunningAgent, StoppingWorker, - UnloadReason, WorkerCommand, WorkerInstance, complete_stopping_worker, + PendingLiveInvocationDisposition, RelinquishReason, RetryDecision, RunningAgent, + StoppingWorker, UnloadReason, WorkerCommand, WorkerInstance, complete_stopping_worker, }; use crate::workerctx::default::Context; use golem_common::model::AgentInvocationKind; use golem_common::model::agent::AgentMode; use golem_common::model::oplog::AgentError; use golem_common::model::{OplogIndex, Timestamp}; - use golem_service_base::error::worker_executor::WorkerExecutorError; + use golem_service_base::error::worker_executor::{InterruptKind, WorkerExecutorError}; use std::collections::VecDeque; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -3142,6 +3217,46 @@ mod tests { delete(seal(filesystem)).await.unwrap(); } + #[test] + fn exit_owner_failure_prefers_relinquishment() { + let shard_lost = || Some(OwnerFailureWinner::Lifecycle(InterruptKind::ShardLost)); + let recovery_failure = WorkerExecutorError::unknown("recovery failed"); + + assert!(matches!( + exit_owner_failure(shard_lost(), None, None), + OwnerFailureWinner::Lifecycle(InterruptKind::ShardLost) + )); + // A shard that moved outranks both a queued lifecycle interrupt and a recovery failure: + // the agent was neither suspended through the API nor broken, it has a new owner. + assert!(matches!( + exit_owner_failure( + shard_lost(), + Some(InterruptKind::Suspend(Timestamp::now_utc())), + Some(&recovery_failure), + ), + OwnerFailureWinner::Lifecycle(InterruptKind::ShardLost) + )); + + // Without a relinquishment the previous order stands: the queued interrupt, then the + // recovery failure, then an interrupt stamped now. + assert!(matches!( + exit_owner_failure( + None, + Some(InterruptKind::Suspend(Timestamp::now_utc())), + Some(&recovery_failure), + ), + OwnerFailureWinner::Lifecycle(InterruptKind::Suspend(_)) + )); + assert!(matches!( + exit_owner_failure(None, None, Some(&recovery_failure)), + OwnerFailureWinner::Infrastructure(_) + )); + assert!(matches!( + exit_owner_failure(None, None, None), + OwnerFailureWinner::Lifecycle(InterruptKind::Interrupt(_)) + )); + } + impl Drop for TestStoreOwner { fn drop(&mut self) { self.dropped.store(true, Ordering::Release); @@ -3831,6 +3946,39 @@ mod tests { ); } + /// A session completion that failed on a fence must not take the in-place restart: that + /// would reopen the oplog at the epoch the storage just refused. + #[test] + fn a_fenced_session_completion_relinquishes_instead_of_restarting() { + let agent_id = golem_common::model::AgentId { + component_id: golem_common::model::component::ComponentId(uuid::Uuid::from_u128(1)), + agent_id: "fenced".to_string(), + }; + let fence = crate::services::oplog::OplogFence { + agent_id: agent_id.clone(), + expected_epoch: golem_common::model::ShardEpoch(3), + actual_epoch: Some(golem_common::model::ShardEpoch(4)), + }; + let flattened = WorkerExecutorError::runtime("durable stream commit failed"); + + assert!(matches!( + session_completion_relinquishment(&flattened, Some(fence.clone())), + Some(RelinquishReason::Fenced(Some(latched))) if *latched == fence + )); + assert!(matches!( + session_completion_relinquishment( + &WorkerExecutorError::OplogFenced { + agent_id, + expected_epoch: 3, + actual_epoch: Some(4), + }, + None, + ), + Some(RelinquishReason::Fenced(None)) + )); + assert!(session_completion_relinquishment(&flattened, None).is_none()); + } + #[test] fn durable_live_streaming_invocation_always_reconstructs_the_store() { assert_eq!( diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index beae704bcc..3ccfa2fb03 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -134,8 +134,8 @@ use golem_common::model::worker::{ use golem_common::model::{ AgentFingerprint, AgentId, AgentInvocation, AgentInvocationOutput, AgentInvocationPayload, AgentInvocationResult, AgentMetadata, AgentStatusRecord, IdempotencyKey, OwnedAgentId, - PendingInvocationRef, PendingUpdateKind, PendingUpdateRef, ShardEpoch, ShardId, Timestamp, - TimestampedAgentInvocation, + PendingInvocationRef, PendingUpdateKind, PendingUpdateRef, ShardAssignment, ShardEpoch, + ShardId, Timestamp, TimestampedAgentInvocation, }; use golem_common::one_shot::OneShotEvent; use golem_common::read_only_lock; @@ -735,18 +735,48 @@ impl Worker { .unwrap_or_else(|| "-".to_string()) } - /// Records that this executor is giving the agent up. Idempotent; the first reason wins. + /// Records that this executor is giving the agent up. Idempotent; the first reason wins and is + /// the only one logged. /// /// Synchronous and lock-free on purpose: the stop path calls it while holding the instance - /// lock, where anything that could take that lock again would deadlock. + /// lock, where anything that could take that lock again would deadlock. Logging takes no + /// instance lock. pub(crate) fn mark_relinquished(&self, reason: RelinquishReason) -> bool { - self.relinquishment.set(reason).is_ok() + let first = self.relinquishment.set(reason).is_ok(); + if first && let Some(reason) = self.relinquishment.get() { + // Debug rather than warn: the oplog that latched a fence has already warned with both + // epochs, and a revoke or reassignment is logged by the sweep that gives agents up. + debug!( + agent_id = %self.owned_agent_id, + ?reason, + "Giving the agent up: this executor no longer owns its shard" + ); + } + first } pub(crate) fn is_relinquished(&self) -> bool { self.relinquishment.get().is_some() } + /// Stops this worker through this handle, whichever generation it is. Nothing public reaches + /// the stop through an arbitrary handle, and a handle kept past its generation is exactly what + /// the tests using this need. + #[cfg(feature = "test-utils")] + #[doc(hidden)] + pub async fn test_stop(&self) { + self.stop_internal( + false, + None, + UnloadRequest::ordinary(UnloadReason::ExplicitStop), + FinalWorkerState::Unloaded { + startup_failure: None, + }, + PendingLiveInvocationDisposition::Fail, + ) + .await; + } + /// What anyone waiting on this agent is told. Every variant is one the worker service answers /// by refreshing its routing table and retrying, so the invocation lands on the new owner /// rather than failing. @@ -758,6 +788,14 @@ impl Worker { }) } + /// What entity bodies are torn down with once this agent has been given up; `None` while it is + /// still this executor's. + pub(crate) fn relinquished_owner_failure(&self) -> Option { + self.relinquishment + .get() + .map(RelinquishReason::owner_failure) + } + /// Give the agent up: stop it here without writing to its oplog or its status, and drop it /// from this executor so the worker service resumes it on the shard's owner. /// @@ -784,13 +822,25 @@ impl Worker { .await; } + /// Whether `cell` is this worker's published status. Each generation shares its cell with its + /// own worker-state actor and nothing else, so the cell identifies the generation to code that + /// holds it but not the worker. + pub(crate) fn shares_status_cell( + &self, + cell: &Arc>, + ) -> bool { + Arc::ptr_eq(&self.last_known_status, cell) + } + pub(crate) async fn remove_from_active_agents(&self) { match self.relinquishment.get() { + // Scoped to this generation: a relinquished agent passes through here more than once, + // and no repeat pass may evict the generation that replaced it. Some(reason) => { self.deps .active_agents() - .remove_with(&self.owned_agent_id, reason.owner_failure()) - .await + .remove_generation(self, reason.owner_failure()) + .await; } None => self.deps.active_agents().remove(&self.owned_agent_id).await, } @@ -3874,7 +3924,9 @@ impl Worker { producer .append_session_record(StreamSessionRecordV1::Prepared(prepared.clone())) .await - .map_err(|error| WorkerExecutorError::invalid_request(error.to_string()))?; + .map_err(|error| { + error.into_worker_executor_error(WorkerExecutorError::invalid_request) + })?; prepared } else { let pending = self @@ -3913,7 +3965,7 @@ impl Worker { }, ) .await - .map_err(|error| WorkerExecutorError::runtime(error.to_string()))?; + .map_err(|error| error.into_worker_executor_error(WorkerExecutorError::runtime))?; attached_during_prepare = true; prepared }; @@ -4287,7 +4339,12 @@ impl Worker { accepted_epoch, }) .await - .map_err(|error| WorkerExecutorError::invalid_request(error.to_string()))?; + .map_err(|error| match self.oplog.fence() { + // The attempt reports its errors as text. A refused append behind one has already + // latched the fence, which is reported as such so the caller reroutes to the owner. + Some(fence) => WorkerExecutorError::from(OplogError::Fenced(fence)), + None => WorkerExecutorError::invalid_request(error), + })?; let streams = make_streams(accepted_epoch, attempt.attempt_id)?; for mapping in &mappings { @@ -5211,6 +5268,50 @@ impl Worker { result } + /// Adds and commits an entry that gates a side effect - a durable scope `Start` or a remote + /// transaction's begin - and reports it as fenced when the storage refused it. + /// + /// `add_and_commit_oplog` cannot be used for these. Below the commit threshold an add only + /// buffers, so it answers with an index even on an oplog whose fence has latched, and the + /// status actor answers the refused commit that follows by spawning the relinquish rather + /// than failing. The caller would run its side effect for an entry that never reached the + /// storage, and the shard's new owner, finding no `Start`, would run it again. + /// + /// Every other storage failure keeps the fail-stop behaviour of `add_to_oplog_or_relinquish`. + pub async fn add_and_commit_oplog_or_fenced( + &self, + entry: OplogEntry, + ) -> Result { + let index = match self.oplog.add(entry).await { + Ok(index) => index, + Err(OplogError::Fenced(fence)) => { + self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence.clone())))); + return Err(OplogError::Fenced(fence)); + } + Err(error) => panic!("oplog write: {error}"), + }; + self.commit_oplog_or_fenced(CommitLevel::Always).await?; + Ok(index) + } + + /// Commits the buffered entries, reporting a refused commit as fenced; for a commit a side + /// effect waits on (see [`Self::add_and_commit_oplog_or_fenced`]). + /// + /// The fence is read from the oplog's latch rather than from the commit, which swallows it. + /// That read is not early: the refused append latches the fence before the status actor + /// replies, and this awaits the reply. + pub async fn commit_oplog_or_fenced( + &self, + commit_level: CommitLevel, + ) -> Result { + let index = self.commit_oplog_and_update_state(commit_level).await; + let written = written_unless_fenced(index, self.oplog.fence()); + if let Err(OplogError::Fenced(fence)) = &written { + self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence.clone())))); + } + written + } + pub async fn queue_card_revocation(&self, card_id: CardId) -> Option { self.queue_card_revocations(&[card_id]) .await @@ -5313,6 +5414,8 @@ impl Worker { } let boundary_guard = self.card_event_boundary_lock.clone().lock_owned().await; + // A refused entry is not delivered: the sender learns the shard moved instead of treating + // a transfer this oplog never recorded as received. self.state_actor .append_and_commit_attached( OplogEntry::card_event_queued( @@ -5323,7 +5426,7 @@ impl Worker { instance_guard, boundary_guard, ) - .await; + .await?; Ok(()) } @@ -5826,10 +5929,24 @@ impl Worker { self.handle_stop_result(stop_result).await; - // The single removal point. Every loop exit and every external stop passes through here, - // so a relinquished agent is dropped from this executor exactly once - and only after the - // loop has actually gone, so the new owner cannot recover it while it is still running. - if self.is_relinquished() { + // The removal point. Every loop exit and every external stop passes through here, so a + // relinquished agent arrives more than once: from its own loop, again from the relinquish + // that waited for that loop, and from any stop that arrives through a handle kept past its + // generation. Everything below is scoped to this generation; a pass that finds the entry + // gone or holding a newer generation does nothing. It runs only after the loop has gone, so + // the new owner cannot recover the agent while it is still running here. + // + // Waiters are failed here as well, in memory only. An agent given up from inside its own + // loop - a fence refused in a host call traps with `ShardLost` - stops without failing + // anyone, and while this executor's assignment still names the shard their ownership + // re-check keeps passing. The relinquish spawned by the loop's exit commit answers them + // only if it still finds this generation cached, and the loop's own removal can get there + // first. Failing them before the removal, while this generation still holds the entry, + // keeps the failure away from a newer generation's waiters, which match by agent id. Keys + // that already have a result keep it. A generation that left the cache some other way + // first (an idle expiry, an environment unload) is not reached here. + if self.is_relinquished() && self.deps.active_agents().is_cached_generation(self).await { + self.fail_pending_invocations(self.relinquish_error()).await; self.remove_from_active_agents().await; } @@ -6304,10 +6421,13 @@ impl Worker { freshness_disposition: InvocationFreshnessDisposition, ) -> Result { // Captured once, here, and cached for the life of the oplog. One live oplog is one - // ownership generation: a renewal never moves an epoch, and when one does move this - // executor is the side that lost the shard, so re-reading it per write would only let a - // losing executor talk itself back into ownership. - let shard_epoch = owned_shard_epoch(this, &owned_agent_id.agent_id); + // ownership generation. Re-reading the epoch per write would only let a losing executor + // talk itself back into ownership. A renewal never moves an epoch. A delivery that raises + // the epoch of a shard this executor kept means the shard left and came back: the + // assignment sweep gives the agent up and the open-oplog cache declines the old handle, so + // the next open claims the new epoch. Read before anything else, so a worker whose shard + // has already left the assignment is refused without touching storage. + let shard_epoch = owned_shard_epoch(this, &owned_agent_id.agent_id)?; let component_id = owned_agent_id.component_id(); // KnownFresh has already been validated against the ephemeral agent type, phantom ID, and @@ -6948,15 +7068,98 @@ struct PendingWorkerInterrupt { unload_request: UnloadRequest, } -/// The shard epoch this executor currently holds for the agent's shard, if it holds one. +/// Whether an entry the oplog answered with `index` was written, given the fence the oplog has +/// latched by the time its commit returned. A latched fence means the commit was refused, whatever +/// the add answered, so the index is not handed to a caller about to run a side effect. +fn written_unless_fenced( + index: OplogIndex, + latched: Option, +) -> Result { + match latched { + Some(fence) => Err(OplogError::Fenced(fence)), + None => Ok(index), + } +} + +/// The shard epoch the agent's oplog is opened to assert, read from this executor's current +/// assignment. See [`shard_epoch_to_assert`]. +fn owned_shard_epoch( + this: &T, + agent_id: &AgentId, +) -> Result, WorkerExecutorError> { + shard_epoch_to_assert( + this.shard_service().try_get_current_assignment().as_ref(), + agent_id, + ) +} + +/// The shard epoch an oplog opened for `agent_id` asserts under `assignment`. /// -/// `None` only when there is no assignment at all yet (before registration), or when the agent's -/// shard is not in it - in which case admission has already refused the work, and an oplog opened -/// without an epoch simply asserts nothing. -fn owned_shard_epoch(this: &T, agent_id: &AgentId) -> Option { - let assignment = this.shard_service().try_get_current_assignment()?; +/// `Ok(None)` only when there is no assignment at all, before the first registration. +/// +/// An assignment that does not hold the agent's shard is refused with `ShardingNotReady`, which +/// the worker service answers by refreshing its routing and retrying. It does not map to `None`, +/// because admission and this read take separate locks. A revoke can land between them, and its +/// sweep cannot see a worker that is still being built. That worker would otherwise open an oplog +/// that asserts nothing and stay cached, unfenced, across a later re-grant of the shard. +/// +/// The same refusal covers a cleared assignment (lapsed lease, deregistration), which holds no +/// shards at all. +fn shard_epoch_to_assert( + assignment: Option<&ShardAssignment>, + agent_id: &AgentId, +) -> Result, WorkerExecutorError> { + let Some(assignment) = assignment else { + return Ok(None); + }; + let shard_id = ShardId::from_agent_id(agent_id, assignment.number_of_shards); + assignment + .epoch_of(&shard_id) + .map(Some) + .ok_or(WorkerExecutorError::ShardingNotReady) +} + +/// Whether an agent whose oplog asserts `held` has been superseded by a delivery that assigns its +/// shard at `assigned`. +/// +/// - A shard's epoch rises only when it changed owner in between. A kept shard at a higher epoch +/// therefore means another executor may have written to the agent. +/// - An equal epoch is the same ownership generation. +/// - A lower epoch never comes from a newer owner, because the shard manager never lowers an +/// epoch. Giving the agent up would only reopen it below the epoch its oplog row already holds. +/// - `None` on either side is not this rule's business. An ephemeral handle asserts nothing, and +/// an absent shard is the membership check's to handle. +pub(crate) fn epoch_superseded(held: Option, assigned: Option) -> bool { + matches!((held, assigned), (Some(held), Some(assigned)) if held < assigned) +} + +/// Whether a delivered `assignment` takes `agent_id` away from this executor. `held` is the epoch +/// the agent's oplog asserts, `None` when it asserts none or the agent has no oplog yet. +/// +/// True when either: +/// - the assignment does not hold the agent's shard. No assignment at all holds nothing, the same +/// answer `ShardService::check_worker` gives. +/// - it holds the shard at a higher epoch than `held`, per [`epoch_superseded`]. The shard left and +/// came back, so another executor may have written to the agent in between. +/// +/// Every sweep of a delivered assignment selects with this one predicate. A caller that checks +/// agents still being created passes `None`, which leaves only the membership test. +/// +/// Membership and epochs only, never the lease. A lapsed lease refuses new work and leaves running +/// work alone. +pub(crate) fn relinquished_by_assignment( + assignment: Option<&ShardAssignment>, + agent_id: &AgentId, + held: Option, +) -> bool { + let Some(assignment) = assignment else { + return true; + }; let shard_id = ShardId::from_agent_id(agent_id, assignment.number_of_shards); - assignment.epoch_of(&shard_id) + match assignment.epoch_of(&shard_id) { + None => true, + assigned => epoch_superseded(held, assigned), + } } /// Why this executor is giving an agent up: it no longer owns the agent's shard. @@ -6971,7 +7174,9 @@ pub(crate) enum RelinquishReason { Fenced(Option>), /// The shard manager revoked the shard. ShardRevoked, - /// A delivered assignment no longer contains the agent's shard. + /// A delivered assignment no longer holds the agent's shard, or holds it at a higher epoch + /// than the agent's oplog asserts. In the second case the shard came back to this executor, + /// and the agent is reopened here at the new epoch. ShardNotAssigned, } @@ -8386,6 +8591,106 @@ mod tests { use std::path::Path; use test_r::test; + /// The add that buffered a side effect's entry answered with an index. A fence latched by the + /// commit must win over that answer, or the side effect runs for an entry nobody can see. + #[test] + fn a_latched_fence_refuses_an_entry_its_add_accepted() { + let fence = OplogFence { + agent_id: AgentId { + component_id: ComponentId(Uuid::new_v4()), + agent_id: "gated".to_string(), + }, + expected_epoch: ShardEpoch(8), + actual_epoch: Some(ShardEpoch(9)), + }; + let index = OplogIndex::from_u64(5); + + assert_eq!(written_unless_fenced(index, None), Ok(index)); + assert_eq!( + written_unless_fenced(index, Some(fence.clone())), + Err(OplogError::Fenced(fence)) + ); + } + + /// Admission and the epoch read are not atomic. An agent whose shard left the assignment in + /// between must be refused, not handed an oplog that asserts nothing. + #[test] + fn an_agent_whose_shard_left_the_assignment_is_refused_an_epoch() { + let agent = AgentId { + component_id: ComponentId(Uuid::new_v4()), + agent_id: "fenced".to_string(), + }; + + assert!(matches!(shard_epoch_to_assert(None, &agent), Ok(None))); + assert!(matches!( + shard_epoch_to_assert( + Some(&ShardAssignment::unexpiring(1, [ShardId::new(0)])), + &agent + ), + Ok(Some(ShardEpoch(0))) + )); + assert!(matches!( + shard_epoch_to_assert(Some(&ShardAssignment::unexpiring(1, [])), &agent), + Err(WorkerExecutorError::ShardingNotReady) + )); + } + + #[test] + fn a_kept_shard_supersedes_an_agent_only_when_its_epoch_rose() { + assert!(epoch_superseded(Some(ShardEpoch(0)), Some(ShardEpoch(1)))); + assert!(!epoch_superseded(Some(ShardEpoch(1)), Some(ShardEpoch(1)))); + // An equal-revision redelivery carrying a lower epoch must not give up a newer handle. + assert!(!epoch_superseded(Some(ShardEpoch(1)), Some(ShardEpoch(0)))); + assert!(!epoch_superseded(None, Some(ShardEpoch(1)))); + assert!(!epoch_superseded(Some(ShardEpoch(0)), None)); + } + + #[test] + fn a_delivery_relinquishes_agents_off_its_shards_or_behind_their_shards_epoch() { + let agent = AgentId { + component_id: ComponentId(Uuid::new_v4()), + agent_id: "swept".to_string(), + }; + let at_epoch = |epoch: u64| ShardAssignment { + shard_epochs: HashMap::from([(ShardId::new(0), ShardEpoch(epoch))]), + ..ShardAssignment::unexpiring(1, []) + }; + + // Membership: no assignment, or one without the shard, gives the agent up whatever its + // oplog asserts. + for held in [None, Some(ShardEpoch(0))] { + assert!(relinquished_by_assignment(None, &agent, held)); + assert!(relinquished_by_assignment( + Some(&ShardAssignment::unexpiring(1, [])), + &agent, + held + )); + } + + // A kept shard gives the agent up only when its epoch rose past the one the oplog asserts. + assert!(!relinquished_by_assignment( + Some(&at_epoch(1)), + &agent, + Some(ShardEpoch(1)) + )); + assert!(relinquished_by_assignment( + Some(&at_epoch(1)), + &agent, + Some(ShardEpoch(0)) + )); + assert!(!relinquished_by_assignment( + Some(&at_epoch(0)), + &agent, + Some(ShardEpoch(1)) + )); + // An agent asserting nothing, such as one still being created, is judged by membership. + assert!(!relinquished_by_assignment( + Some(&at_epoch(1)), + &agent, + None + )); + } + #[test] fn pending_manual_update_keeps_storage_key_but_has_no_semantic_key() { let target_revision = ComponentRevision::new(2).unwrap(); diff --git a/golem-worker-executor/src/worker/state_actor.rs b/golem-worker-executor/src/worker/state_actor.rs index eae7661679..22fe145fa8 100644 --- a/golem-worker-executor/src/worker/state_actor.rs +++ b/golem-worker-executor/src/worker/state_actor.rs @@ -112,12 +112,14 @@ enum StatusJob { }, /// Appends an entry and completes its commit + fold transaction even if the caller is /// cancelled. The caller-acquired guards remain owned by this job until the transaction ends. + /// Replies with the refusal when the oplog has a new owner, so the caller never reports an + /// entry as delivered that was not written. AppendAndCommitAttached { entry: Box, _worker_keepalive: Arc, _instance_guard: OwnedMutexGuard, _card_event_boundary_guard: OwnedMutexGuard<()>, - done: oneshot::Sender<()>, + done: oneshot::Sender>, }, AppendInvocationIfVersion { entry: Box, @@ -238,7 +240,12 @@ impl WorkerStateActor { } => { complete_status_job( async { - let changed = state.commit_and_update_state(level, committed).await; + // The reply keeps its shape: these callers learn of a fence from + // the oplog's latch and from the relinquish the refusal spawned. + let changed = state + .commit_and_update_state(level, committed) + .await + .unwrap_or(false); let index = state.oplog.current_oplog_index().await; (index, changed) }, @@ -257,15 +264,20 @@ impl WorkerStateActor { async { match state.oplog.add(*entry).await { Ok(_) => { - state + if let Err(fence) = state .commit_and_update_state(CommitLevel::Always, None) - .await; + .await + { + return Err(OplogError::Fenced(fence)); + } state.ensure_status_attached().await; + Ok(()) } // The shard has a new owner: give the agent up and leave no // further trace in an oplog that is no longer ours. Err(OplogError::Fenced(fence)) => { - state.relinquish_fenced_agent(fence) + state.relinquish_fenced_agent(fence.clone()); + Err(OplogError::Fenced(fence)) } Err(error) => panic!("oplog write: {error}"), } @@ -303,9 +315,16 @@ impl WorkerStateActor { } return Err(error); } - state + // The entry is only buffered until this commit, which is where a + // takeover is found. The key has not reached the status, so nothing + // that fails pending invocations can answer its caller: the enqueue + // itself has to be refused. + if let Err(fence) = state .commit_and_update_state(CommitLevel::Always, None) - .await; + .await + { + return Err(OplogError::Fenced(fence)); + } if let WorkerInstance::Running(running) = &*instance_guard { running.sender.send(WorkerCommand::WorkAvailable).unwrap(); } @@ -426,7 +445,7 @@ impl WorkerStateActor { worker: Arc>, instance_guard: OwnedMutexGuard, card_event_boundary_guard: OwnedMutexGuard<()>, - ) { + ) -> Result<(), OplogError> { let worker_keepalive: Arc = worker; self.commit .run_status_job(|done| StatusJob::AppendAndCommitAttached { @@ -601,16 +620,23 @@ impl StatusState { /// discipline while still dropping the agent from this executor - which a bare /// `mark_relinquished` would not do, because on a background path nothing else is unwinding /// to carry the stop out. + /// + /// Only the generation this actor belongs to is given up, identified by the status cell the + /// two share. By the time the task runs that generation may be gone and a newer one cached + /// under the same id, which is left alone: at a stale epoch its own open latches the fence and + /// gives it up, and at a re-granted epoch it is legitimately this executor's. fn relinquish_fenced_agent(&self, fence: OplogFence) { let active_agents = self.deps.active_agents(); - let agent_id = self.owned_agent_id.agent_id.clone(); + let owned_agent_id = self.owned_agent_id.clone(); + let status_cell = self.last_known_status.clone(); tokio::spawn(async move { - active_agents - .relinquish_matching( - RelinquishReason::Fenced(Some(Box::new(fence))), - |candidate| candidate == &agent_id, - ) - .await; + if let Some(worker) = active_agents.try_get_cached(&owned_agent_id).await + && worker.shares_status_cell(&status_cell) + { + worker + .relinquish(RelinquishReason::Fenced(Some(Box::new(fence)))) + .await; + } }); } @@ -618,19 +644,23 @@ impl StatusState { /// committed entries into the published status or marks the status detached when it can no /// longer be incrementally computed (e.g. after a revert or a snapshot update). Returns /// whether the published status (or its detachment) changed. + /// + /// A commit the storage refused because the shard has a new owner is returned as the fence + /// rather than folded into "unchanged": a caller whose entry was only buffered until this + /// commit must not report it as written. async fn commit_and_update_state( &self, commit_level: CommitLevel, committed: Option>, - ) -> bool { + ) -> Result { let new_entries = match self.oplog.commit(commit_level).await { Ok(entries) => entries, Err(OplogError::Fenced(fence)) => { // Nothing was committed and nothing more can be. The `committed` sender is // dropped rather than signalled: a fenced commit is not a commit, and every // awaiter already reads a dropped sender as "no commit observed". - self.relinquish_fenced_agent(fence); - return false; + self.relinquish_fenced_agent(fence.clone()); + return Err(fence); } Err(error) => panic!("oplog write: {error}"), }; @@ -703,11 +733,14 @@ impl StatusState { .fetch_add(authority_change_count, Ordering::Release); } - changed + Ok(changed) } async fn reattach(&self) { - self.commit_and_update_state(CommitLevel::Always, None) + // A refused commit has already given the agent up; the status is still recomputed from + // what was committed before it. + let _ = self + .commit_and_update_state(CommitLevel::Always, None) .await; self.ensure_status_attached().await; diff --git a/golem-worker-executor/tests/active_agents.rs b/golem-worker-executor/tests/active_agents.rs index 8ce0586af6..f8d6724ee1 100644 --- a/golem-worker-executor/tests/active_agents.rs +++ b/golem-worker-executor/tests/active_agents.rs @@ -204,6 +204,98 @@ async fn a_revoke_older_than_the_last_delivery_does_not_sweep_agents( Ok(()) } +/// A delivery that keeps a shard but raises its epoch means the shard left this executor and +/// came back, so another executor may have written to its agents in between. An agent still +/// holding the older epoch's oplog must be given up and reopened at the new epoch; a delivery at +/// the epoch it already holds must leave it alone. +/// +/// Driven over the wire for the same reason as the stale-revoke test above: the sweep lives in +/// the gRPC handler. +#[test] +#[timeout("120s")] +#[tracing::instrument] +async fn a_delivery_that_raises_a_kept_shards_epoch_gives_its_agents_up( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + + let component = executor + .component_dep(&context.default_environment_id, host_api_tests) + .store() + .await?; + let parsed_agent_id = agent_id!("Clock", "epoch-raise-owner"); + let agent_id = executor + .start_agent(&component.id, parsed_agent_id.clone()) + .await?; + executor + .invoke_and_await_agent(&component, &parsed_agent_id, "healthcheck", data_value!()) + .await?; + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &agent_id); + assert!(executor.worker_is_loaded(&owned_agent_id).await); + + // The single-shard bootstrap holds shard 0 at epoch 0, so the agent's oplog asserts epoch 0. + let shard = ShardId { value: 0 }; + let mut client = executor.client.clone(); + let push = |epoch: u64, revision: u64| AssignShardsRequest { + shard_epochs: vec![ShardEpochEntry { + shard_id: Some(shard), + epoch, + }], + revision, + number_of_shards: 1, + }; + + // The handler sweeps on every applied push, changed or not, so this does run the sweep. + let same_epoch = client.assign_shards(push(0, 5)).await?.into_inner(); + assert!(matches!( + same_epoch.result, + Some(assign_shards_response::Result::Success(_)) + )); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + executor.worker_is_loaded(&owned_agent_id).await, + "a push at the epoch the agent already holds must not give it up" + ); + + let raised = client.assign_shards(push(1, 6)).await?.into_inner(); + assert!(matches!( + raised.result, + Some(assign_shards_response::Result::Success(_)) + )); + // The agent is idle, so assignment recovery does not track it and cannot reopen it while + // this waits. + wait_until("the superseded agent to be given up", || async { + !executor.worker_is_loaded(&owned_agent_id).await + }) + .await?; + + executor + .invoke_and_await_agent(&component, &parsed_agent_id, "healthcheck", data_value!()) + .await?; + assert!(executor.worker_is_loaded(&owned_agent_id).await); + + // The reopen asserts epoch 1. Had it been handed the epoch-0 handle back, this push would + // sweep it as superseded. + let kept = client.assign_shards(push(1, 7)).await?.into_inner(); + assert!(matches!( + kept.result, + Some(assign_shards_response::Result::Success(_)) + )); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + executor.worker_is_loaded(&owned_agent_id).await, + "the agent reopened after the raise must hold the new epoch and survive a push of it" + ); + + drop(client); + drop(executor); + Ok(()) +} + #[test] #[timeout("120s")] #[tracing::instrument] diff --git a/golem-worker-executor/tests/api.rs b/golem-worker-executor/tests/api.rs index 79a20c1eef..8a76af42c8 100644 --- a/golem-worker-executor/tests/api.rs +++ b/golem-worker-executor/tests/api.rs @@ -48,7 +48,7 @@ use golem_worker_executor::worker::INVOCATION_OWNERSHIP_RECHECK_INTERVAL; use golem_worker_executor_test_utils::{ LastUniqueId, PrecompiledComponent, TestContext, TestExecutorOverrides, TestWorkerExecutor, WorkerExecutorTestDependencies, fake_ownership, registry_test_card, start, start_customized, - start_with_overrides, start_with_redis_storage, + start_with_overrides, start_with_redis_storage, take_agent_oplog_over_at_epoch, }; use pretty_assertions::assert_eq; use redis::Commands; @@ -6160,6 +6160,297 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away_while_the_even Ok(()) } +/// A stop that reaches a generation already given up must leave the generation that replaced it +/// alone. +/// +/// A relinquished agent passes through the stop's removal more than once - from its own loop and +/// again from the relinquish that waited for it - and a handle kept past its generation can stop it +/// once more. Removal used to be keyed by agent id only, so any of those passes evicted whatever +/// was cached under that id by then: here, the newer generation the shard's return created. +#[test] +#[tracing::instrument] +#[timeout("2m")] +async fn a_stop_through_a_relinquished_generation_leaves_the_next_generation_cached( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + _tracing: &Tracing, + #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + let component = executor + .component_dep(&context.default_environment_id, host_api_tests) + .store() + .await?; + let agent_id = agent_id!("Clocks", "stale-generation-stop"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &worker_id); + + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + let stale = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is not cached after its first invocation"))? + .primary(); + + revoke_shard_zero(&executor).await?; + tokio::time::timeout(Duration::from_secs(10), async { + while executor.worker_is_cached(&owned_agent_id).await { + sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| anyhow!("the agent stayed cached after its shard was revoked"))?; + + assign_shard_zero(&executor).await?; + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + let fresh = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is not cached after the shard came back"))? + .primary(); + assert!( + !Arc::ptr_eq(&stale, &fresh), + "the shard's return must have created a new generation, or this test proves nothing" + ); + + // The stale generation is already unloaded and relinquished, so this goes straight to the + // removal. + stale.test_stop().await; + + assert!( + executor.worker_is_cached(&owned_agent_id).await, + "a stop through the relinquished generation evicted the generation that replaced it" + ); + let cached = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is no longer cached"))? + .primary(); + assert!( + Arc::ptr_eq(&cached, &fresh), + "the cached generation changed under a stop through a stale handle" + ); + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + Ok(()) +} + +/// A caller awaiting an invocation whose oplog write was refused inside a host call must be told +/// to reroute. +/// +/// A fence found in a host call surfaces as a `ShardLost` trap: the agent marks itself +/// relinquished and its loop stops without failing anyone. The executor's own assignment still +/// names the shard - this is the zombie, and nobody has told it - so the waiter's ownership +/// re-check keeps passing. Two things can answer the caller: the relinquish spawned when the loop's +/// exit commit is refused, and the loop's own stop. The spawned one misses the caller whenever the +/// loop removes the agent before it looks, and nothing in a test can hold it back, so this pins +/// the outcome rather than that ordering. +/// +/// The takeover lands while the guest is parked in `poll` inside `sleep_for`, after +/// `subscribe_duration` has committed through the status actor. The first write after it is then +/// the gated monotonic-clock hook's own commit when the guest reads the elapsed time: a refusal in +/// the host call, not in the status actor. +#[test] +#[tracing::instrument] +#[timeout("2m")] +async fn a_caller_waiting_on_an_invocation_fenced_inside_a_host_call_is_told_to_reroute( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + _tracing: &Tracing, + #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, +) -> anyhow::Result<()> { + use golem_common::model::oplog::PublicOplogEntry; + + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + let component = executor + .component_dep(&context.default_environment_id, host_api_tests) + .store() + .await?; + let agent_id = agent_id!("Clocks", "fenced-inside-a-host-call"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &worker_id); + + let mut caller = { + let executor = executor.clone(); + let component = component.clone(); + let agent_id = agent_id.clone(); + tokio::spawn( + async move { + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(6.0f64)) + .await + } + .in_current_span(), + ) + }; + executor + .wait_for_status(&worker_id, AgentStatus::Running, Duration::from_secs(10)) + .await?; + + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let oplog = executor.get_oplog(&worker_id, OplogIndex::INITIAL).await?; + let subscribed = oplog.iter().any(|entry| match &entry.entry { + PublicOplogEntry::End(end) => oplog.iter().any(|start| { + start.oplog_index == end.start_index + && matches!( + &start.entry, + PublicOplogEntry::Start(params) + if params.function_name == "monotonic_clock::subscribe_duration" + ) + }), + _ => false, + }); + if subscribed { + return anyhow::Ok(()); + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .map_err(|_| anyhow!("the guest never subscribed to its sleep"))??; + // Lets `subscribe_duration`'s own commit finish, so the guest is parked in `poll` with about + // five seconds of sleep left and nothing else is due to write. + sleep(Duration::from_secs(1)).await; + + let mut clock = executor + .gate_next_monotonic_clock_start(&owned_agent_id) + .await?; + take_agent_oplog_over_at_epoch(deps, &context, &owned_agent_id, 1).await?; + + let answer = match tokio::time::timeout(Duration::from_secs(30), &mut caller).await { + Ok(joined) => joined?, + Err(_) => { + caller.abort(); + bail!( + "the caller was never answered, although its agent's oplog has a new owner and \ + this executor gave the agent up" + ); + } + }; + info!(result = ?answer, "caller was answered"); + let error = + answer.expect_err("the invocation's writes were refused, so it cannot have a value"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("ShardingNotReady") || rendered.contains("Sharding not ready"), + "the caller has to be given the error worker-service reroutes on; instead it got: \ + {rendered}" + ); + + tokio::time::timeout(Duration::from_secs(10), async { + while executor.worker_is_cached(&owned_agent_id).await { + sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| anyhow!("the fenced agent stayed cached on this executor"))?; + + // The hook signals `entered` only once its commit has succeeded, and by now the agent is gone + // from this executor, so a gate that was going to be entered already has been. Silence means + // the refusal happened at that commit, inside the host call. + assert!( + tokio::time::timeout(Duration::from_millis(500), clock.entered()) + .await + .is_err(), + "the gated clock commit went through, so the refusal was not the host call's" + ); + Ok(()) +} + +/// An invocation enqueued onto an oplog that has a new owner must be refused, not accepted. +/// +/// Enqueueing buffers the pending-invocation entry and commits it through the status actor, and +/// that commit is where the takeover is found. The refusal used to be folded into "status +/// unchanged", so the enqueue reported success for a key that never reached the status. Neither the +/// relinquish the refusal spawns nor the stop's removal fails keys the status does not hold, so +/// nothing ever answered the caller. +#[test] +#[tracing::instrument] +#[timeout("2m")] +async fn an_invocation_enqueued_onto_a_fenced_oplog_is_refused_rather_than_accepted( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + _tracing: &Tracing, + #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + let component = executor + .component_dep(&context.default_environment_id, host_api_tests) + .store() + .await?; + let agent_id = agent_id!("Clocks", "fenced-enqueue"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &worker_id); + + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + let idle = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is not cached after its first invocation"))? + .primary(); + + take_agent_oplog_over_at_epoch(deps, &context, &owned_agent_id, 1).await?; + + // The refusal has to come from the enqueue's own commit. Had anything written to the idle + // agent first, the relinquish that write spawned could evict it, and the invocation would then + // be refused at a fresh generation's open instead, which proves nothing about the enqueue. + let cached = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the idle agent left the cache before the enqueue reached it"))? + .primary(); + assert!( + Arc::ptr_eq(&idle, &cached), + "the generation that opened the oplog at the old epoch is no longer the cached one" + ); + + let answer = tokio::time::timeout( + Duration::from_secs(10), + executor.invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)), + ) + .await + .map_err(|_| { + anyhow!( + "the caller was never answered: the invocation was accepted onto an oplog whose \ + pending entry the storage refused" + ) + })?; + info!(result = ?answer, "caller was answered"); + let error = answer.expect_err("the pending entry was never committed, so there is no value"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("ShardingNotReady") || rendered.contains("Sharding not ready"), + "the caller has to be given the error worker-service reroutes on; instead it got: \ + {rendered}" + ); + + tokio::time::timeout(Duration::from_secs(10), async { + while executor.worker_is_cached(&owned_agent_id).await { + sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| anyhow!("the fenced agent stayed cached on this executor"))?; + Ok(()) +} + /// The test executor runs `ShardManagerServiceSingleShard`, so every agent in /// these tests lives on shard 0. Moving that one shard moves all of them. async fn revoke_shard_zero(executor: &TestWorkerExecutor) -> anyhow::Result<()> { diff --git a/golem-worker-executor/tests/indexed_storage.rs b/golem-worker-executor/tests/indexed_storage.rs index 6a0c5a3cb9..f2454fe023 100644 --- a/golem-worker-executor/tests/indexed_storage.rs +++ b/golem-worker-executor/tests/indexed_storage.rs @@ -350,121 +350,127 @@ async fn postgres_singleton_append_many_preserves_storage_contract( ) { let storage = storage.get_indexed_storage().await; let value = Bytes::from_static(&[0, 255, 17, 3]); - for ns in [primary, compressed] { - storage - .append_many( - "svc", - "api", - "entity", - &ns.ns, - "singleton", - Arc::from([]), - None, - ) - .await - .unwrap(); - assert!( - !storage - .exists("svc", "api", ns.ns.clone(), "singleton") - .await - .unwrap() - ); - storage - .append_many( - "svc", - "api", - "entity", - &ns.ns, - "singleton", - Arc::from([(17, value.clone())]), - None, - ) - .await - .unwrap(); - assert_eq!( + // Once asserting no epoch, which is a lone autocommit INSERT, and once asserting the recorded + // one, which goes through the fenced transaction. A caller must not be able to tell the two + // paths apart. + for (key, shard_epoch) in [ + ("singleton", None), + ("fenced-singleton", Some(ShardEpoch(1))), + ] { + for ns in [primary, compressed] { + if let Some(epoch) = shard_epoch { + storage + .upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, epoch) + .await + .unwrap(); + } storage - .read("svc", "api", "entity", ns.ns.clone(), "singleton", 0, 100) + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([]), + shard_epoch, + ) .await - .unwrap(), - vec![(17, value.to_vec())] - ); - assert_eq!( + .unwrap(); + assert!( + !storage + .exists("svc", "api", ns.ns.clone(), key) + .await + .unwrap() + ); storage - .length("svc", "api", ns.ns.clone(), "singleton") + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(17, value.clone())]), + shard_epoch, + ) .await - .unwrap(), - 1 - ); - assert_eq!( - storage - .last("svc", "api", "entity", ns.ns.clone(), "singleton") + .unwrap(); + assert_eq!( + storage + .read("svc", "api", "entity", ns.ns.clone(), key, 0, 100) + .await + .unwrap(), + vec![(17, value.to_vec())] + ); + assert_eq!( + storage + .length("svc", "api", ns.ns.clone(), key) + .await + .unwrap(), + 1 + ); + assert_eq!( + storage + .last("svc", "api", "entity", ns.ns.clone(), key) + .await + .unwrap(), + Some((17, value.to_vec())) + ); + let (_, keys) = storage + .scan( + "svc", + "api", + ns.meta.clone(), + Some(key), + ScanCursor::default(), + 10, + ) .await - .unwrap(), - Some((17, value.to_vec())) - ); - let (_, keys) = storage - .scan( - "svc", - "api", - ns.meta.clone(), - Some("singleton"), - ScanCursor::default(), - 10, - ) - .await - .unwrap(); - assert_eq!(keys, vec!["singleton".to_string()]); + .unwrap(); + assert_eq!(keys, vec![key.to_string()]); + assert!(matches!( + storage + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(u64::MAX, value.clone())]), + shard_epoch, + ) + .await, + Err(IndexedStorageError::Other(_)) + )); + assert_eq!( + storage + .length("svc", "api", ns.ns.clone(), key) + .await + .unwrap(), + 1 + ); + } assert!(matches!( storage .append_many( "svc", "api", "entity", - &ns.ns, - "singleton", - Arc::from([(u64::MAX, value.clone())]), - None, + &primary.ns, + key, + Arc::from([(17, Bytes::from_static(b"replacement"))]), + shard_epoch, ) .await, - Err(IndexedStorageError::Other(_)) + Err(IndexedStorageError::Conflict(_)) )); assert_eq!( storage - .length("svc", "api", ns.ns.clone(), "singleton") + .read("svc", "api", "entity", primary.ns.clone(), key, 0, 100) .await .unwrap(), - 1 + vec![(17, value.to_vec())] ); } - assert!(matches!( - storage - .append_many( - "svc", - "api", - "entity", - &primary.ns, - "singleton", - Arc::from([(17, Bytes::from_static(b"replacement"))]), - None, - ) - .await, - Err(IndexedStorageError::Conflict(_)) - )); - assert_eq!( - storage - .read( - "svc", - "api", - "entity", - primary.ns.clone(), - "singleton", - 0, - 100 - ) - .await - .unwrap(), - vec![(17, value.to_vec())] - ); } #[test] @@ -1672,6 +1678,69 @@ async fn a_stale_epoch_append_is_refused_and_writes_nothing( } } +#[test] +#[tracing::instrument] +async fn an_append_ahead_of_the_recorded_epoch_is_refused( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let fencing = is.expects_fencing(); + let is = is.get_indexed_storage().await; + let key = "fence-ahead"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(5)) + .await + .unwrap(); + + // The check is equality, not "at least": a record behind the asserted epoch means the open + // that should have raised it never ran, so the write is not ours to make. Both call shapes, so + // a single-entry path that parts from the batch path cannot quietly relax the check. + let batch = is + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([ + (1, Bytes::from_static(b"a")), + (2, Bytes::from_static(b"b")), + (3, Bytes::from_static(b"c")), + ]), + Some(ShardEpoch(6)), + ) + .await; + let batch_length = is.length("svc", "api", ns.ns.clone(), key).await.unwrap(); + let single = is + .append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 4, + b"d".to_vec(), + Some(ShardEpoch(6)), + ) + .await; + let length = is.length("svc", "api", ns.ns.clone(), key).await.unwrap(); + + if fencing { + assert_fenced(batch, 6, Some(5)); + assert_eq!( + batch_length, 0, + "a refused batch must leave no entry behind" + ); + assert_fenced(single, 6, Some(5)); + assert_eq!(length, 0, "a refused append must leave no entry behind"); + } else { + batch.unwrap(); + single.unwrap(); + assert_eq!(length, 4); + } +} + #[test] #[tracing::instrument] async fn an_append_without_a_recorded_epoch_is_refused( @@ -1846,6 +1915,48 @@ async fn deleting_the_recorded_epoch_fences_later_writes( } } +#[test] +#[tracing::instrument] +async fn a_deleted_record_does_not_remember_its_epoch( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + let is = is.get_indexed_storage().await; + let key = "fence-forgotten"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(9)) + .await + .unwrap(); + is.delete_oplog_metadata("svc", "api", ns.ns.clone(), key) + .await + .unwrap(); + + // The documented limit of the fence: the forward-only rule lives on the record, so once the + // record is gone a lower epoch than the one it held is recorded and written through. Closing + // this needs the delete to keep the epoch, which changes this test on purpose. + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(8)) + .await + .unwrap(); + is.append( + "svc", + "api", + "entity", + ns.ns.clone(), + key, + 1, + b"a".to_vec(), + Some(ShardEpoch(8)), + ) + .await + .unwrap(); + + assert_eq!( + is.length("svc", "api", ns.ns.clone(), key).await.unwrap(), + 1 + ); +} + #[test] #[tracing::instrument] async fn the_backend_reports_whether_it_fences( diff --git a/golem-worker-service/src/service/worker/client.rs b/golem-worker-service/src/service/worker/client.rs index 61833056b0..2f1fbb3752 100644 --- a/golem-worker-service/src/service/worker/client.rs +++ b/golem-worker-service/src/service/worker/client.rs @@ -64,7 +64,6 @@ use golem_service_base::grpc::client::MultiTargetGrpcClient; use golem_service_base::model::auth::AuthCtx; use golem_service_base::model::{ComponentFileSystemNode, GetOplogResponse}; use golem_service_base::service::routing_table::{HasRoutingTableService, RoutingTableService}; -use std::future::Future; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::{collections::HashMap, sync::Arc}; @@ -90,26 +89,59 @@ fn freshness_disposition_for_dispatch( pub type InvocationRequestStream = Pin + Send + 'static>>; pub type InvocationResponseStream = Pin> + Send + 'static>>; -type InvocationSessionCall<'a> = Pin< - Box< - dyn Future>, Status>> - + Send - + 'a, - >, ->; - -fn invoke_agent_session_once<'a>( - client: &'a mut WorkerExecutorClient>, - request: Option, -) -> InvocationSessionCall<'a> { - match request { - Some(request) => Box::pin(client.invoke_agent_session(request)), - None => Box::pin(std::future::ready(Err(Status::aborted( - "invocation session request was already consumed", - )))), +/// One dispatch of an invocation session's opening frame, read up to the executor's decision. +#[derive(Debug)] +struct SessionDispatch { + decision: Option, + responses: tonic::Streaming, + /// Feeds the executor the rest of the caller's request stream. Nothing is sent on it before + /// the executor accepts, and a dispatch that is not accepted drops it unused. + tail: mpsc::Sender, +} + +impl SessionDispatch { + fn routing_miss(&self) -> Option<&InvocationRejected> { + match &self.decision { + Some(InvocationResponse { + response: Some(invocation_response::Response::Rejected(rejected)), + }) if rejected.reason == InvocationRejectionReason::ShardingNotReady as i32 => { + Some(rejected) + } + _ => None, + } + } + + fn accepted(&self) -> bool { + matches!( + self.decision, + Some(InvocationResponse { + response: Some(invocation_response::Response::Accepted(_)), + }) + ) } } +async fn dispatch_invocation_session( + client: &mut WorkerExecutorClient>, + first: InvocationRequest, +) -> Result { + let (tail, receiver) = mpsc::channel(1); + let mut responses = client + .invoke_agent_session( + futures::stream::once(std::future::ready(first)).chain(ReceiverStream::new(receiver)), + ) + .await? + .into_inner(); + // `tail` stays open while the decision is awaited: an executor rejects a request stream that + // closes before acceptance as a protocol violation. + let decision = responses.message().await?; + Ok(SessionDispatch { + decision, + responses, + tail, + }) +} + #[derive(Debug)] enum OneShotInvocationSessionResult { Success(AgentInvocationOutput), @@ -1774,6 +1806,12 @@ impl WorkerClient for WorkerExecutorWorkerClient { if rejected.reason == InvocationRejectionReason::ShardingNotReady as i32 => { + // The typed error is bare on purpose; the rejection text is what names the + // agent and, for a fenced oplog, both epochs, so it is kept in the log. + tracing::debug!( + error = %rejected.error, + "Executor turned the invocation away as a routing miss" + ); Err(WorkerExecutorError::ShardingNotReady.into()) } OneShotInvocationSessionResult::Rejected(rejected) => { @@ -1798,41 +1836,81 @@ impl WorkerClient for WorkerExecutorWorkerClient { agent_id: &AgentId, request: InvocationRequestStream, ) -> WorkerResult { - let routing_table = self - .routing_table_service - .get_routing_table() - .await - .map_err(|error| { - WorkerServiceError::InternalCallError( - CallWorkerExecutorError::FailedToGetRoutingTable(error), - ) - })?; - let pod = routing_table.lookup(agent_id).ok_or_else(|| { - WorkerServiceError::InternalCallError(CallWorkerExecutorError::FailedToConnectToPod( - Status::unavailable(format!("no active shard for agent {agent_id}")), - )) + // An executor that does not own the agent's shard rejects the session before accepting + // it, and no input may precede acceptance. So the opening frame alone is dispatched, and + // is dispatched again after the routing table is refreshed, exactly like the unary path; + // the caller's input is attached only to the executor that accepted. A transport failure + // before the decision arrives also sends the opening frame again, with the same attempt + // id and downgraded to MayExist, as the unary path does. Two consequences: input a caller + // sends too early is held until acceptance, so it is the response validator rather than + // the executor that refuses it; and while no executor owns the shard the session waits, + // which keeps a WebSocket session's connection open for as long as that lasts. + let mut request = request; + let first = request.next().await.ok_or_else(|| { + WorkerServiceError::Internal( + "invocation session request ended before start".to_string(), + ) })?; - let request = Arc::new(std::sync::Mutex::new(Some(request))); - let response = self - .worker_executor_clients - .call_without_retry( + let first_dispatch = Arc::new(AtomicBool::new(true)); + + let dispatch = self + .call_worker_executor( + agent_id.clone(), "invoke_agent_session", - pod.uri(self.worker_executor_clients.uses_tls()), move |worker_executor_client| { - let request = request - .lock() - .unwrap_or_else(|poison| poison.into_inner()) - .take(); - invoke_agent_session_once(worker_executor_client, request) + let mut first = first.clone(); + if let Some(invocation_request::Request::Start(start)) = &mut first.request + && start.freshness_disposition + == golem_api_grpc::proto::golem::worker::InvocationFreshnessDisposition::KnownFresh + as i32 + && freshness_disposition_for_dispatch( + InvocationFreshnessDisposition::KnownFresh, + &first_dispatch, + ) == InvocationFreshnessDisposition::MayExist + { + start.freshness_disposition = + golem_api_grpc::proto::golem::worker::InvocationFreshnessDisposition::MayExist + as i32; + } + Box::pin(dispatch_invocation_session(worker_executor_client, first)) }, + |dispatch| match dispatch.routing_miss() { + Some(rejected) => { + tracing::debug!( + error = %rejected.error, + "Executor turned the invocation session away as a routing miss" + ); + Err(ResponseMapResult::ShardingNotReady) + } + None => Ok(dispatch), + }, + WorkerServiceError::InternalCallError, ) - .await - .map_err(|status| { - WorkerServiceError::InternalCallError( - CallWorkerExecutorError::FailedToConnectToPod(status), - ) - })?; - Ok(Box::pin(response.into_inner())) + .await?; + + if dispatch.accepted() { + let tail = dispatch.tail; + tokio::spawn(async move { + loop { + tokio::select! { + item = request.next() => match item { + Some(item) => { + if tail.send(item).await.is_err() { + break; + } + } + None => break, + }, + // The session ended on the executor's side while the caller still had + // input to send; stop holding the caller's stream. + _ = tail.closed() => break, + } + } + }); + } + Ok(Box::pin( + futures::stream::iter(dispatch.decision.map(Ok)).chain(dispatch.responses), + )) } async fn control_durable_stream_attachment( @@ -2464,7 +2542,7 @@ mod rejection_mapping_tests { use super::{ WorkerClient, WorkerExecutorWorkerClient, WorkerServiceError, decode_invocation_rejection, }; - use futures::{Stream, stream}; + use futures::{Stream, StreamExt, stream}; use golem_api_grpc::proto::golem::schema::{SchemaValue, schema_value}; use golem_api_grpc::proto::golem::shardmanager::{ IpAddress, Pod as GrpcPod, RoutingTable as GrpcRoutingTable, RoutingTableEntry, ShardId, @@ -2472,8 +2550,9 @@ mod rejection_mapping_tests { }; use golem_api_grpc::proto::golem::worker::v1::{AgentError, agent_error}; use golem_api_grpc::proto::golem::worker::{ + InputStreamEnd, InvocationAccepted, InvocationFreshnessDisposition as WireFreshness, InvocationRejected, InvocationRejectionReason, InvocationRequest, InvocationResponse, - invocation_response, + InvocationStart, invocation_request, invocation_response, }; use golem_api_grpc::proto::golem::workerexecutor::v1::worker_executor_server::{ WorkerExecutor, WorkerExecutorServer, @@ -2493,12 +2572,15 @@ mod rejection_mapping_tests { use golem_service_base::model::auth::AuthCtx; use golem_service_base::model::quota_lease::{PendingReservation, QuotaLease}; use golem_service_base::service::routing_table::{RoutingTableConfig, RoutingTableService}; + use std::collections::BTreeMap; use std::net::Ipv4Addr; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; use test_r::test; use tokio::net::TcpListener; + use tokio_stream::wrappers::ReceiverStream; use tokio_stream::wrappers::TcpListenerStream; use tonic::codec::CompressionEncoding; use tonic::{Request, Response, Status}; @@ -2552,6 +2634,10 @@ mod rejection_mapping_tests { _port: u16, _pod_name: Option, _executor_id: uuid::Uuid, + _previous_shard_epochs: std::collections::BTreeMap< + golem_common::model::ShardId, + ShardEpoch, + >, ) -> Result { unreachable!() } @@ -2560,6 +2646,10 @@ mod rejection_mapping_tests { &self, _executor_id: uuid::Uuid, _shard_epochs: std::collections::BTreeMap, + _fenced_shard_epochs: std::collections::BTreeMap< + golem_common::model::ShardId, + ShardEpoch, + >, ) -> Result { unreachable!() } @@ -2612,11 +2702,18 @@ mod rejection_mapping_tests { } /// Rejects every invocation as `NotFound`, after first rejecting `routing_misses` of them as - /// having reached an executor that does not own the agent's shard. + /// having reached an executor that does not own the agent's shard. With `accept` it accepts + /// instead of rejecting as `NotFound`, and keeps the session open until its request stream ends. #[derive(Clone, Default)] struct RejectingExecutor { routing_misses: Arc, + accept: bool, calls: Arc, + /// The opening frame's freshness disposition, one per call in call order. + dispositions: Arc>>, + /// Receives `(call index, frames after the opening one)` once a call's request stream has + /// ended, which happens after the call returned its response stream. + tail_frames: Option>, } macro_rules! unimplemented_unary { @@ -2759,46 +2856,82 @@ mod rejection_mapping_tests { ) -> Result, Status> { let mut requests = request.into_inner(); let start = requests.message().await?.expect("missing invocation start"); - let (idempotency_key, agent_id) = match start.request { + let (idempotency_key, agent_id, freshness_disposition) = match start.request { Some(golem_api_grpc::proto::golem::worker::invocation_request::Request::Start( start, - )) => (start.idempotency_key, start.agent_id), + )) => ( + start.idempotency_key, + start.agent_id, + start.freshness_disposition, + ), other => panic!("expected invocation start, got {other:?}"), }; - self.calls.fetch_add(1, Ordering::SeqCst); + let call = self.calls.fetch_add(1, Ordering::SeqCst); + self.dispositions + .lock() + .unwrap() + .push(freshness_disposition); let routing_miss = self .routing_misses .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| { left.checked_sub(1) }) .is_ok(); - let (reason, error) = if routing_miss { - ( - InvocationRejectionReason::ShardingNotReady, - "0 is not in shards []", - ) + let response = if !routing_miss && self.accept { + invocation_response::Response::Accepted(InvocationAccepted { + agent_id, + idempotency_key, + ..Default::default() + }) } else { - (InvocationRejectionReason::NotFound, "agent not found") + let (reason, error) = if routing_miss { + ( + InvocationRejectionReason::ShardingNotReady, + "0 is not in shards []", + ) + } else { + (InvocationRejectionReason::NotFound, "agent not found") + }; + invocation_response::Response::Rejected(InvocationRejected { + reason: reason as i32, + error: error.to_string(), + idempotency_key, + agent_id, + component_revision: None, + }) }; - Ok(Response::new(Box::pin(stream::iter([Ok( - InvocationResponse { - response: Some(invocation_response::Response::Rejected( - InvocationRejected { - reason: reason as i32, - error: error.to_string(), - idempotency_key, - agent_id, - component_revision: None, - }, - )), - }, - )])))) + let accepted = matches!(response, invocation_response::Response::Accepted(_)); + + let (responses, receiver) = tokio::sync::mpsc::channel(1); + responses + .send(Ok(InvocationResponse { + response: Some(response), + })) + .await + .expect("the response stream was dropped before it was returned"); + // A rejected session ends at once, as an executor's does. An accepted one stays open + // until the caller's request stream ends, or its forwarded input could be cut off. + let held_open = accepted.then_some(responses); + let tail_frames = self.tail_frames.clone(); + tokio::spawn(async move { + let mut frames = 0; + // An error ends the count as well: a caller drops a rejected session's request + // stream rather than finishing it. + while let Ok(Some(_)) = requests.message().await { + frames += 1; + } + if let Some(tail_frames) = tail_frames { + let _ = tail_frames.send((call, frames)); + } + drop(held_open); + }); + Ok(Response::new(Box::pin(ReceiverStream::new(receiver)))) } } - /// Invokes an agent through a worker service whose only executor is `executor`, and returns - /// the error the invocation ends with. - async fn invoke_against(executor: RejectingExecutor) -> WorkerServiceError { + /// A worker service client whose only executor is `executor`, and an agent to invoke through + /// it. + async fn client_against(executor: RejectingExecutor) -> (WorkerExecutorWorkerClient, AgentId) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); tokio::spawn(async move { @@ -2847,6 +2980,13 @@ mod rejection_mapping_tests { component_id: ComponentId::new(), agent_id: "missing".to_string(), }; + (client, agent_id) + } + + /// Invokes an agent through a worker service whose only executor is `executor`, and returns + /// the error the invocation ends with. + async fn invoke_against(executor: RejectingExecutor) -> WorkerServiceError { + let (client, agent_id) = client_against(executor).await; client .invoke_agent( @@ -2906,6 +3046,126 @@ mod rejection_mapping_tests { "the retried call's answer must be the one surfaced, got {public_error:?}" ); } + + fn session_start(agent_id: &AgentId) -> InvocationRequest { + InvocationRequest { + request: Some(invocation_request::Request::Start(InvocationStart { + agent_id: Some(agent_id.clone().into()), + method_name: Some("run".to_string()), + idempotency_key: Some( + golem_common::model::IdempotencyKey::new("session-key".to_string()).into(), + ), + freshness_disposition: WireFreshness::KnownFresh as i32, + ..Default::default() + })), + } + } + + #[test] + async fn a_routing_miss_rejection_on_a_session_is_retried_on_the_shard_owner() { + // A streaming session meets the same routing miss as a unary invocation and has to be + // retried the same way, including giving up KnownFresh once a dispatch may have reached + // an executor. + let executor = RejectingExecutor { + routing_misses: Arc::new(AtomicUsize::new(1)), + ..Default::default() + }; + let calls = executor.calls.clone(); + let dispositions = executor.dispositions.clone(); + let (client, agent_id) = client_against(executor).await; + + let responses = client + .invoke_agent_session( + &agent_id, + Box::pin(stream::iter([session_start(&agent_id)])), + ) + .await + .expect("the session was not dispatched"); + let responses = + tokio::time::timeout(Duration::from_secs(30), responses.collect::>()) + .await + .expect("the session's response stream never ended"); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "the routing miss must be retried, exactly once" + ); + assert_eq!( + *dispositions.lock().unwrap(), + vec![ + WireFreshness::KnownFresh as i32, + WireFreshness::MayExist as i32 + ] + ); + match responses.as_slice() { + [ + Ok(InvocationResponse { + response: Some(invocation_response::Response::Rejected(rejected)), + }), + ] => assert_eq!( + rejected.reason, + InvocationRejectionReason::NotFound as i32, + "the retried call's answer must be the one surfaced" + ), + other => panic!("expected only the retried call's rejection, got {other:?}"), + } + } + + #[test] + async fn session_input_reaches_only_the_executor_that_accepted() { + // No input may precede acceptance, so the executor that turned the session away must have + // seen the opening frame alone, and the caller's input must reach the one that accepted. + let (tail_frames, mut ended) = tokio::sync::mpsc::unbounded_channel(); + let executor = RejectingExecutor { + routing_misses: Arc::new(AtomicUsize::new(1)), + accept: true, + tail_frames: Some(tail_frames), + ..Default::default() + }; + let (client, agent_id) = client_against(executor).await; + let request = stream::iter([ + session_start(&agent_id), + InvocationRequest { + request: Some(invocation_request::Request::InputEnd( + InputStreamEnd::default(), + )), + }, + ]); + + // Held until the end: dropping it would cancel the session before its input arrived. + let mut responses = client + .invoke_agent_session(&agent_id, Box::pin(request)) + .await + .expect("the session was not dispatched"); + let first = tokio::time::timeout(Duration::from_secs(30), responses.next()) + .await + .expect("the session sent no first frame") + .expect("the session's response stream ended without a frame"); + assert!( + matches!( + first, + Ok(InvocationResponse { + response: Some(invocation_response::Response::Accepted(_)), + }) + ), + "the caller must see the acceptance first, got {first:?}" + ); + + let mut frames_per_call = BTreeMap::new(); + while frames_per_call.len() < 2 { + let (call, frames) = tokio::time::timeout(Duration::from_secs(30), ended.recv()) + .await + .expect("a dispatch's request stream never ended") + .expect("the executor stopped reporting"); + frames_per_call.insert(call, frames); + } + assert_eq!( + frames_per_call, + BTreeMap::from([(0, 0), (1, 1)]), + "input must reach only the executor that accepted" + ); + } } #[cfg(test)] diff --git a/integration-tests/tests/sharding.rs b/integration-tests/tests/sharding.rs index d0f09824d8..4556da5b15 100644 --- a/integration-tests/tests/sharding.rs +++ b/integration-tests/tests/sharding.rs @@ -22,6 +22,8 @@ mod tests { use bytes::Bytes; use golem_api_grpc::proto::golem::worker; use golem_client::api::RegistryServiceClient; + #[cfg(unix)] + use golem_common::model::AgentId; use golem_common::model::base64::Base64; use golem_common::model::component::ComponentDto; use golem_common::model::environment_plugin_grant::EnvironmentPluginGrantCreation; @@ -29,7 +31,8 @@ mod tests { use golem_common::model::plugin_registration::{ OplogProcessorPluginSpec, PluginRegistrationCreation, PluginSpecDto, }; - use golem_common::model::{AgentId, AgentStatus, IdempotencyKey, OplogIndex}; + use golem_common::model::{AgentStatus, IdempotencyKey, OplogIndex}; + #[cfg(unix)] use golem_common::schema::SchemaValue; use golem_common::tracing::{TracingConfig, init_tracing_with_default_debug_env_filter}; use golem_common::{agent_id, data_value}; @@ -72,9 +75,6 @@ mod tests { pub async fn create_deps() -> EnvBasedTestDependencies { let deps = EnvBasedTestDependencies::new(EnvBasedTestDependenciesConfig { number_of_shards_override: Some(16), - // The shortest lease the shard manager accepts, so that a paused executor is seen to - // lose its shards well inside a test's timeout. - shard_lease_duration_override: Some(Duration::from_secs(30)), ..EnvBasedTestDependenciesConfig::new() }) .await @@ -281,11 +281,13 @@ mod tests { chaos.await.unwrap(); } + // Pausing an executor is SIGSTOP. + #[cfg(unix)] #[test] - #[timeout(300000)] + #[timeout(360000)] // Not `#[flaky]`, unlike the scenarios above: what this pins is that a duplicate never // happens, and a retry would hide one that only happens some of the time. - async fn an_executor_paused_past_its_lease_cannot_finish_the_invocations_it_started( + async fn an_executor_paused_until_its_shards_move_cannot_finish_the_invocations_it_started( deps: &EnvBasedTestDependencies, cluster_control: &WorkerExecutorClusterControlStub, _tracing: &Tracing, @@ -315,6 +317,24 @@ mod tests { agents.push((parsed_agent_id, agent_id)); } + // Read before anything is in flight, so that connecting does not eat into the delay the + // executors have to be frozen inside. + let pool = match deps.rdb().info() { + DbInfo::Postgres(pg) => sqlx::PgPool::connect(&pg.public_connection_string()) + .await + .expect("Failed to connect to Postgres"), + _ => panic!("this test only implements reading the stored owning epochs from Postgres"), + }; + let mut initial_epochs = Vec::new(); + for (parsed_agent_id, agent_id) in &agents { + let epoch = stored_owning_epoch(&pool, agent_id) + .await + .unwrap_or_else(|| { + panic!("{parsed_agent_id}: the owning epoch is recorded before the first entry") + }); + initial_epochs.push(epoch); + } + let mut invocations = JoinSet::new(); for (parsed_agent_id, _) in &agents { let user = deps.admin().await; @@ -350,14 +370,25 @@ mod tests { cluster_control.pause(*idx).await; } - // Past the frozen executors' leases, so that their shards are granted to the survivor at a - // higher epoch and it recovers and finishes their invocations itself, and past the delay, - // so that the frozen executors' own sleeps are over the moment they wake. - tokio::time::sleep(Duration::from_secs(45)).await; + // Long enough for the shard manager to have taken the frozen executors' shards away and + // granted them to the survivor at a higher epoch, so that it recovers and finishes their + // invocations itself, and past the delay, so that the frozen executors' own sleeps are over + // the moment they wake. What normally moves the shards is the shard manager's health check, + // which unregisters an executor once its probes and their retries have gone unanswered, + // usually well before its lease runs out; and the worker service's keep-alive drops the + // calls stuck on a frozen executor, so that they are retried against the new owner. The + // lease is only the upper bound on the move: at the default 60s, renewed every 20s and + // reaped on a 20s tick, an unrenewed lease is gone within 80s of the pause. That bound + // keeps a thawed executor from waking up as the owner; it does not stretch the callers' + // retries, which count on the health check. + tokio::time::sleep(Duration::from_secs(90)).await; // Thawed, the frozen executors carry on from exactly where they stopped, still holding // invocations the survivor now owns. For each one, either the executor gives the agent up // when it re-registers, or it finishes the sleep first and the fence refuses its write. + // Which of the two each agent took is not visible from here, and both are safe only + // because the survivor recorded a higher epoch first, which the epoch check below asserts. + // The refusal itself is pinned deterministically by golem-worker-executor's oplog tests. info!("Resuming worker executors {frozen:?}"); for idx in frozen { cluster_control.resume(*idx).await; @@ -380,17 +411,28 @@ mod tests { ); } - // No executor may have died on the way: a write the fence let through would land in an - // oplog the survivor is writing too, and a conflicting append there is fatal to the - // executor that makes it - which can be the rightful owner. Checked through the health - // endpoint rather than process liveness: an aborting process stays "running" until the OS - // has finished writing its crash report, which can outlast this test. - for idx in cluster_control.started_indices().await { + let mut owner_changed = false; + for ((parsed_agent_id, agent_id), initial) in agents.iter().zip(&initial_epochs) { + let current = stored_owning_epoch(&pool, agent_id) + .await + .unwrap_or_else(|| { + panic!("{parsed_agent_id}: the owning epoch is no longer recorded") + }); assert!( - cluster_control.is_serving(idx).await, - "worker executor {idx} stopped serving during the test" + current >= *initial, + "{parsed_agent_id}: the stored epoch went down from {initial} to {current}" ); + owner_changed |= current > *initial; } + pool.close().await; + assert!( + owner_changed, + "no agent's shard changed owner during the test, so the run exercised neither the \ + relinquish nor the fence" + ); + + assert_every_started_executor_serves(cluster_control, "while the invocations finished") + .await; for (parsed_agent_id, agent_id) in &agents { assert_eq!( @@ -411,11 +453,53 @@ mod tests { "{parsed_agent_id}: the delayed increment must not have been applied twice" ); } + + // Again after the follow-up calls: they are the first new traffic after the thaw, and the + // first chance for a re-registered executor to recover agents on a shard it was given back. + assert_every_started_executor_serves( + cluster_control, + "while the agents were invoked again", + ) + .await; + } + + /// Asserts that no started executor has died: a write the fence let through would land in an + /// oplog the survivor is writing too, and a conflicting append there is fatal to the executor + /// that makes it - which can be the rightful owner. Checked through the health endpoint rather + /// than process liveness: an aborting process stays "running" until the OS has finished + /// writing its crash report, which can outlast this test. Each call is a single probe, so it + /// only covers what happened before it. + #[cfg(unix)] + async fn assert_every_started_executor_serves( + cluster_control: &WorkerExecutorClusterControlStub, + during: &str, + ) { + for idx in cluster_control.started_indices().await { + assert!( + cluster_control.is_serving(idx).await, + "worker executor {idx} stopped serving {during}" + ); + } + } + + /// The owning epoch the executors' indexed storage holds for `agent_id`'s oplog. Read from the + /// storage itself because the public oplog does not carry the shard epoch. + #[cfg(unix)] + async fn stored_owning_epoch(pool: &sqlx::PgPool, agent_id: &AgentId) -> Option { + sqlx::query_scalar( + "SELECT epoch FROM golem_worker_executor_indexed.oplog_metadata \ + WHERE namespace = 'durable-worker-oplog' AND key = $1", + ) + .bind(agent_id.to_redis_key()) + .fetch_optional(pool) + .await + .expect("Failed to read oplog_metadata") } /// Counts the completions `agent_id`'s oplog records for `method`. Scoped to one method on /// purpose: an agent's own initialization completes asynchronously after `start_agent` returns, /// so any count taken over every method races it. + #[cfg(unix)] async fn count_completions_of(user: &impl TestDsl, agent_id: &AgentId, method: &str) -> usize { user.get_oplog(agent_id, OplogIndex::INITIAL) .await diff --git a/local-run/start.sh b/local-run/start.sh index e6538a930b..4832826a51 100644 --- a/local-run/start.sh +++ b/local-run/start.sh @@ -16,7 +16,9 @@ fi LOCAL_RUN_DIR="${GOLEM_DIR}/local-run" -rm -rf "${LOCAL_RUN_DIR}/data/shard-manager" +# Wipe the executor's indexed storage along with the shard manager's state: the oplog epochs it +# records were minted by that state and are ahead of everything a fresh one mints. +rm -rf "${LOCAL_RUN_DIR}/data/shard-manager" "${LOCAL_RUN_DIR}/data/worker-executor" mkdir -pv "${LOCAL_RUN_DIR}/data/redis" "${LOCAL_RUN_DIR}/data/shard-manager" "${LOCAL_RUN_DIR}/data/worker-executor" "${LOCAL_RUN_DIR}/logs" # start redis From 0ad03ca611702bc6c4ef50e6560ef33f1c69cafa Mon Sep 17 00:00:00 2001 From: Aditya Salunkhe Date: Wed, 16 Sep 2026 19:52:34 +0530 Subject: [PATCH 3/6] Update the shard ownership tests for relinquish-on-revoke --- golem-worker-executor-test-utils/src/lib.rs | 23 +++ golem-worker-executor/tests/api.rs | 154 ++++++++++++++++---- 2 files changed, 147 insertions(+), 30 deletions(-) diff --git a/golem-worker-executor-test-utils/src/lib.rs b/golem-worker-executor-test-utils/src/lib.rs index 54314f550d..557661547c 100644 --- a/golem-worker-executor-test-utils/src/lib.rs +++ b/golem-worker-executor-test-utils/src/lib.rs @@ -5526,6 +5526,11 @@ struct FakeOwnershipState { /// arrived yet fails, with the `Unknown` that `sharding_not_ready_error` /// produces. That must never be read as "the agent moved". assignment_missing: AtomicBool, + /// Report every check the way an executor whose shard has moved away reports + /// it, without a revoke ever arriving. A revoke gives the agent up here and + /// answers its callers directly, so it is the wrong instrument for a test + /// aimed at the periodic ownership re-check. + agent_moved: AtomicBool, /// Make the next check announce itself, wait, and only then report that the /// agent is not ours. hold_next_check: AtomicBool, @@ -5558,6 +5563,16 @@ impl ShardService for FakeOwnership { }); } + if self.state.agent_moved.load(Ordering::SeqCst) { + self.state + .agent_moved_reports + .fetch_add(1, Ordering::SeqCst); + return Err(WorkerExecutorError::invalid_shard_id( + ShardId::new(0), + HashSet::new(), + )); + } + // Taken, not read, so concurrent checks from other calls fall straight // through to the truth while this one is held at the gate. if self.state.hold_next_check.swap(false, Ordering::SeqCst) { @@ -5686,8 +5701,15 @@ impl OwnershipControls { self.state.assignment_missing.store(true, Ordering::SeqCst); } + /// Report every ownership check the way an executor whose shard has moved + /// away reports it. Lasts until [`Self::stop_pretending`]. + pub fn pretend_the_agent_moved(&self) { + self.state.agent_moved.store(true, Ordering::SeqCst); + } + pub fn stop_pretending(&self) { self.state.assignment_missing.store(false, Ordering::SeqCst); + self.state.agent_moved.store(false, Ordering::SeqCst); } /// Hold the next ownership check open, and return once one has arrived. @@ -5730,6 +5752,7 @@ pub fn fake_ownership() -> (TestExecutorOverrides, OwnershipControls) { let state = Arc::new(FakeOwnershipState { assignment_missing: AtomicBool::new(false), + agent_moved: AtomicBool::new(false), hold_next_check: AtomicBool::new(false), assignment_missing_reports: AtomicUsize::new(0), agent_moved_reports: AtomicUsize::new(0), diff --git a/golem-worker-executor/tests/api.rs b/golem-worker-executor/tests/api.rs index 1cd2dc9c53..a70b41ce7e 100644 --- a/golem-worker-executor/tests/api.rs +++ b/golem-worker-executor/tests/api.rs @@ -5821,10 +5821,15 @@ async fn resource_limits_initialized_for_component_owner_not_caller( /// Control for [`a_caller_is_answered_when_its_agents_shard_is_taken_away`]. /// /// An agent's shard leaves this executor and comes straight back, and the -/// promise it is parked on is then completed here. What this pins, and the test -/// below cannot, is that revoking a shard interrupts the agents on it with -/// `InterruptKind::Restart`, and that interrupt on its own does not strand the -/// caller. So the test below is measuring the handoff and not the interrupt. +/// promise it is parked on is then completed here. A revoke gives the agent up +/// rather than restarting it in place - a restart would reopen its oplog at an +/// epoch this executor no longer holds - so the caller is answered at once with +/// an error it can retry, and the shard returning a moment later does not +/// un-answer it. What has to survive the round trip is the work: the agent is +/// this executor's again, the invocation it was running finishes here, and the +/// retry worker-service makes under the same idempotency key is handed that +/// result instead of running it a second time. So the test below is measuring +/// the handoff and not the giving-up. /// /// It does *not* prove the `select!` is cancel-safe, though it did have to stop /// claiming that twice. Only the `wait_for` future is dropped on a tick; @@ -5844,7 +5849,7 @@ async fn a_caller_is_answered_when_its_agents_shard_comes_back( ) -> anyhow::Result<()> { let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await?; - let parked = park_a_caller_on_a_promise( + let mut parked = park_a_caller_on_a_promise( &executor, &context, host_api_tests, @@ -5857,17 +5862,37 @@ async fn a_caller_is_answered_when_its_agents_shard_comes_back( revoke_shard_zero(&executor).await?; assign_shard_zero(&executor).await?; - // Sit here long enough for the caller's ownership re-check to run several - // times before the result exists, so the answer has to survive the re-check - // firing repeatedly and finding nothing wrong. + // Answered by the giving-up, not left to the ownership re-check: the revoke reached this + // executor, so nothing here waits to find out that the agent moved. + let answer = parked + .answer_within( + Duration::from_secs(20), + "caller parked in invoke_and_await was never answered, although the revoke had \ + already given its agent up here", + ) + .await?; + let error = answer.expect_err( + "the agent was given up when its shard was revoked, so the parked call cannot have \ + been handed a value", + ); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("ShardingNotReady") || rendered.contains("Sharding not ready"), + "the caller has to be told to retry rather than handed a failure it would surface to \ + the user; instead it got: {rendered}" + ); + + // Sit here long enough for the re-check to have run several times, so the retry below is + // answered by an agent that survived the window rather than one that happened to be quick. sleep(INVOCATION_OWNERSHIP_RECHECK_INTERVAL * 3).await; parked.complete_the_promise(&executor, vec![42]).await?; let value = parked - .value_within( - Duration::from_secs(30), - "caller was not answered even though the shard came back and the promise \ + .retry_within( + &executor, + Duration::from_secs(60), + "the retry was never answered even though the shard came back and the promise \ was completed on this executor", ) .await?; @@ -5893,9 +5918,13 @@ async fn a_caller_is_answered_when_its_agents_shard_comes_back( /// never disturbed, so nothing below the application layer had anything to /// notice. /// -/// What it should get is an error of the `InvalidShardId` family, which is -/// already what worker-service needs to invalidate its routing table and retry -/// against the new owner. That path exists and works; nothing used to reach it. +/// What it should get is an error worker-service answers by invalidating its +/// routing table and retrying against the new owner. Two errors carry that +/// meaning, and which one arrives depends on how the agent was lost: a revoke +/// that reaches this executor gives the agent up and answers its callers with +/// `ShardingNotReady` at once, while a shard that moves without a revoke +/// arriving is caught by the periodic ownership re-check, which reports +/// `InvalidShardId`. Either is a retry; silence is not. #[test] #[tracing::instrument] #[timeout("2m")] @@ -5907,7 +5936,7 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away( ) -> anyhow::Result<()> { let context = TestContext::new(last_unique_id); let executor = start(deps, &context).await?; - let parked = + let mut parked = park_a_caller_on_a_promise(&executor, &context, host_api_tests, "promise-shard-taken") .await?; @@ -5934,7 +5963,9 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away( answer.expect_err("nobody completed the promise, so the only honest answer is an error"); let rendered = format!("{error:#}"); assert!( - rendered.contains("InvalidShardId"), + rendered.contains("ShardingNotReady") + || rendered.contains("Sharding not ready") + || rendered.contains("InvalidShardId"), "the caller has to be told the shard moved, because that is what makes \ worker-service invalidate its routing table and retry against the new \ owner; instead it got: {rendered}" @@ -6076,6 +6107,11 @@ async fn a_caller_is_not_given_up_on_while_the_shard_assignment_is_missing( /// The buffer is shrunk to 16 so a burst of 64 every 50ms is enough to keep /// the receiver behind; with the default 100000 the flood would have to be /// that much larger to say the same thing. +/// +/// The agent is moved by [`fake_ownership`] rather than by a real revoke. A +/// revoke that reaches this executor gives the agent up and answers its callers +/// itself, so the re-check this test exists for would never run and the flood +/// would prove nothing. #[test] #[tracing::instrument] #[timeout("2m")] @@ -6086,14 +6122,12 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away_while_the_even #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, ) -> anyhow::Result<()> { let context = TestContext::new(last_unique_id); - let overrides = TestExecutorOverrides { - configure: Some(Arc::new(|config| { - config.limits.invocation_result_broadcast_capacity = 16; - })), - ..Default::default() - }; + let (mut overrides, controls) = fake_ownership(); + overrides.configure = Some(Arc::new(|config| { + config.limits.invocation_result_broadcast_capacity = 16; + })); let executor = start_with_overrides(deps, &context, overrides).await?; - let parked = park_a_caller_on_a_promise( + let mut parked = park_a_caller_on_a_promise( &executor, &context, host_api_tests, @@ -6101,12 +6135,11 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away_while_the_even ) .await?; - // Taken while the agent is still resident; revoking its shard drops it. let events = executor.event_bus(&parked.agent_id).await?; - info!("Revoking the shard for good, then flooding the event bus with somebody else's news"); + info!("Reporting the agent as moved, then flooding the event bus with somebody else's news"); - revoke_shard_zero(&executor).await?; + controls.pretend_the_agent_moved(); let somebody_else = AgentId { component_id: parked.agent_id.component_id, @@ -6129,6 +6162,10 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away_while_the_even } }); + // Left unread while the flood runs, so the buffer overflows under it however + // the runtime schedules the two. Reading it straight away races the flood: + // a reader on another worker thread can keep pace with it and never lag. + sleep(Duration::from_millis(200)).await; let overflowed = tokio::time::timeout(Duration::from_secs(2), probe.wait_for(|_| None::<()>)).await; if !matches!(overflowed, Ok(Err(RecvError::Lagged(_)))) { @@ -6159,6 +6196,13 @@ async fn a_caller_is_answered_when_its_agents_shard_is_taken_away_while_the_even rendered.contains("InvalidShardId"), "the caller has to be told the shard moved; instead it got: {rendered}" ); + // Without this the test also passes when the answer came from somewhere other than the + // re-check, which is the one path the flood is here to starve. + assert!( + controls.agent_moved_reports() >= 1, + "the ownership re-check never ran while the bus was lagging, so nothing here says the \ + deadline arm survives a starved subscription" + ); Ok(()) } @@ -6520,17 +6564,25 @@ async fn park_a_caller_on_a_promise( let promise_data = crate::raw_params(vec![promise_id_value.clone()]); + // Parked under an explicit key so a test can reissue the call the way worker-service reissues + // one it was told to reroute: the retry lands on this same invocation instead of starting a + // second run of it. + let idempotency_key = IdempotencyKey::fresh(); + let executor_clone = executor.clone(); let component_clone = component.clone(); let agent_id_clone = agent_id.clone(); + let key_clone = idempotency_key.clone(); + let params = promise_data.clone(); let fiber = tokio::spawn( async move { executor_clone - .invoke_and_await_agent( + .invoke_and_await_agent_with_key( &component_clone, &agent_id_clone, + &key_clone, "await_promise", - promise_data, + params, ) .await } @@ -6545,6 +6597,10 @@ async fn park_a_caller_on_a_promise( agent_id: worker_id, promise_id: promise_id_value, caller: fiber, + component, + parsed_agent_id: agent_id, + idempotency_key, + promise_data, }) } @@ -6554,13 +6610,18 @@ struct ParkedCaller { agent_id: AgentId, promise_id: SchemaValue, caller: JoinHandle>, + /// What it takes to reissue the parked call under its own idempotency key. + component: ComponentDto, + parsed_agent_id: golem_common::model::agent::ParsedAgentId, + idempotency_key: IdempotencyKey, + promise_data: golem_common::schema::TypedSchemaValue, } impl ParkedCaller { /// Waits for the parked call to come back. The outer result says whether it /// was answered at all; the inner one is what it was told. async fn answer_within( - mut self, + &mut self, patience: Duration, gave_up: &str, ) -> anyhow::Result> { @@ -6593,9 +6654,42 @@ impl ParkedCaller { Ok(()) } + /// Reissues the parked call under its original idempotency key, the way + /// worker-service reissues one an executor told it to reroute, and returns + /// the value that retry is given. + /// + /// The key is what makes this a retry rather than a second run: an + /// invocation already recorded under it is not executed again, so the answer + /// here is the answer to the call that was parked. + async fn retry_within( + &self, + executor: &TestWorkerExecutor, + patience: Duration, + gave_up: &str, + ) -> anyhow::Result { + tokio::time::timeout( + patience, + executor.invoke_and_await_agent_with_key( + &self.component, + &self.parsed_agent_id, + &self.idempotency_key, + "await_promise", + self.promise_data.clone(), + ), + ) + .await + .map_err(|_| anyhow!("{gave_up}"))?? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value")) + } + /// Waits for the parked call and returns the value it was given, failing if /// it was not answered in time or was answered with an error. - async fn value_within(self, patience: Duration, gave_up: &str) -> anyhow::Result { + async fn value_within( + mut self, + patience: Duration, + gave_up: &str, + ) -> anyhow::Result { self.answer_within(patience, gave_up) .await?? .into_return_value() From 36fc0b3114e3a099e4f0b58f1bf3ccb7193350f2 Mon Sep 17 00:00:00 2001 From: Aditya Salunkhe Date: Thu, 17 Sep 2026 17:22:48 +0530 Subject: [PATCH 4/6] Surface refused oplog commits, fail fast after a fence, and relinquish without deadlocks --- .../understanding-durable-execution/SKILL.md | 64 ++- .../reference/crash-matrix.md | 33 +- Makefile.toml | 1 + golem-debugging-service/src/debug_context.rs | 4 +- .../src/quota/quota_service_tests.rs | 46 ++ golem-shard-manager/src/quota/quota_state.rs | 17 +- golem-shard-manager/src/sharding/model.rs | 148 ++++- golem-test-framework/Cargo.toml | 5 + .../src/components/worker_executor/spawned.rs | 53 +- .../tests/signal_unreaped_child.rs | 64 +++ golem-worker-executor-test-utils/src/lib.rs | 8 +- .../src/durable_host/call_coordinator.rs | 20 +- .../durable_host/clocks/monotonic_clock.rs | 2 +- .../src/durable_host/concurrent/call.rs | 20 +- .../src/durable_host/concurrent/tests.rs | 3 +- .../src/durable_host/durability.rs | 66 ++- .../src/durable_host/durable_session.rs | 54 +- .../src/durable_host/golem/retry_api.rs | 4 +- .../src/durable_host/golem/v1x.rs | 6 +- .../src/durable_host/http/types.rs | 14 +- golem-worker-executor/src/durable_host/mod.rs | 187 +++++-- .../src/durable_host/permissions/mod.rs | 14 +- .../src/durable_host/wasm_rpc/mod.rs | 6 +- .../src/services/active_agents/mod.rs | 3 +- .../src/services/oplog/compressed.rs | 121 ++++- .../src/services/oplog/mod.rs | 30 +- .../src/services/oplog/multilayer.rs | 16 + .../src/services/oplog/plugin.rs | 65 ++- .../src/services/oplog/primary.rs | 101 +++- .../src/services/oplog/tests.rs | 221 +++++++- .../src/services/shard_manager.rs | 450 +++++++++++++--- .../src/services/worker_fork.rs | 22 +- .../src/storage/indexed/sqlite.rs | 78 ++- golem-worker-executor/src/worker/instance.rs | 11 +- .../src/worker/invocation_loop.rs | 325 +++++++---- golem-worker-executor/src/worker/lifecycle.rs | 2 +- golem-worker-executor/src/worker/mod.rs | 510 ++++++++++++------ .../src/worker/state_actor.rs | 37 +- .../src/workerctx/default.rs | 4 +- golem-worker-executor/src/workerctx/mod.rs | 10 +- golem-worker-executor/tests/api.rs | 56 +- golem-worker-executor/tests/instance_layer.rs | 18 +- .../worker-executor-walkthrough.html | 13 +- local-run/start.sh | 4 + 44 files changed, 2262 insertions(+), 674 deletions(-) create mode 100644 golem-test-framework/tests/signal_unreaped_child.rs diff --git a/.agents/skills/understanding-durable-execution/SKILL.md b/.agents/skills/understanding-durable-execution/SKILL.md index 599628593b..d5a8898cd1 100644 --- a/.agents/skills/understanding-durable-execution/SKILL.md +++ b/.agents/skills/understanding-durable-execution/SKILL.md @@ -33,9 +33,15 @@ and entity bodies) and `retries.md` (in-function versus trap-based retries). 2. **The resident runtime is disposable.** The Wasmtime `Store`, the worker task, the executor process, sockets, channels and subscriptions can vanish while work is pending. Suspension, - eviction, resharding, restart and crash must all be recoverable the same way: throw the - instance away, build a new `Store`, replay the oplog, continue. Lifecycle hints (`Suspend`, + eviction, restart and crash must all be recoverable the same way, on the same executor: throw + the instance away, build a new `Store`, replay the oplog, continue. Lifecycle hints (`Suspend`, `Interrupted`, `Restart`) change status and scheduling policy, not the recovery mechanism. + Losing the shard does not reconstruct here at all: every oplog write asserts this executor's + shard epoch inside the storage transaction, and once another executor holds the shard, the + write is refused (`OplogError::Fenced`) instead of accepted. The agent then *relinquishes* + (`InterruptKind::ShardLost`) — stopped with nothing more written, dropped from this executor, + never restarted in place — and it is the new owner that builds the `Store` and replays. See + "Resharding, revocation and the oplog epoch fence" below and `crash-matrix.md`. Owner: `worker/invocation_loop.rs::run` (outer loop: create instance → recover → run → suspend/retry) and `durable_host/mod.rs::prepare_instance`. @@ -93,6 +99,14 @@ says how strict that commit is: `Always` waits for durable storage; `DurableOnly for durable agents (`PrimaryOplog::commit` flushes everything; `EphemeralOplog` honours the level). Guarantees such as "accepted only after commit" refer to the commit, not the append. +A commit can also be *refused*: every append and commit asserts this executor's shard epoch +inside the storage transaction, and storage that has recorded a newer epoch returns +`OplogError::Fenced` instead of writing anything. `commit_oplog_and_update_state` and +`add_and_commit_oplog` surface that refusal rather than swallowing it, so a refused +`PendingAgentInvocation` commit is not acknowledged as accepted and a refused +`AgentInvocationFinished` commit is not published to waiters. The first refusal latches — every +later add on that oplog is refused too — and the agent relinquishes instead of retrying. + `worker/state_actor.rs::commit_and_update_state` samples the appended tip before its explicit commit and ignores receipt entries already folded into the published status. Primary/ephemeral threshold flushes and replica waits can commit outside the status actor, so even an empty receipt @@ -142,8 +156,13 @@ each recorded invocation in `InvocationMode::Replay`, and when there is no furth `AgentInvocationStarted` it switches to live. Replay starts from the chosen snapshot baseline (see Snapshots and updates), not necessarily from `OplogIndex::INITIAL`. Interruption kinds (`Worker::set_interrupting`): `Interrupt` stays interrupted, `Restart` is a simulated crash with -automatic recovery, `Suspend` unloads and resumes on demand; all three end in the same -reconstruction path. Eviction (`EvictionClass::{LoadedIdle, WarmRunnable}`) never unloads a +automatic recovery, `Suspend` unloads and resumes on demand — each of these three reconstructs on +this same executor. `ShardLost` does not: it means this executor lost the agent's shard (a +revoked/reassigned shard, or an oplog write refused on the shard epoch), and instead of +reconstructing, the agent relinquishes — stopped without writing to its oplog or status, dropped +from this executor, and left for the new owner to reconstruct. + +Eviction (`EvictionClass::{LoadedIdle, WarmRunnable}`) never unloads a worker that is executing or holds non-durable in-memory work. Ephemeral agents are fail-stop: `reconstructed_ephemeral` rebuilds only for observation and result lookup, "but the instance must never be started again" (`worker/mod.rs`, `INACTIVE_EPHEMERAL_AGENT_ERROR`). @@ -184,6 +203,42 @@ in a pending p3 wait). `Resumed` does not enqueue an invocation: the existing does not use this marker and retains its normal `Idle`/automatic-recovery semantics; ephemeral agents retain their clean fail-stop lifecycle and never append it. +### Resharding, revocation and the oplog epoch fence + +Two triggers give an agent up rather than reconstructing it here, and both end in the same place, +`RelinquishReason` and `InterruptKind::ShardLost`: + +- **Assignment change.** The shard manager's `RevokeShards` and `AssignShards` gRPC calls + (`grpc/mod.rs::revoke_shards_internal`, `::assign_shards_internal` via + `apply_shard_assignment_effects`) relinquish every agent whose shard this executor no longer + holds, or whose held epoch fell behind the delivered one (another executor may have written to + it meanwhile) — `RelinquishReason::ShardRevoked` for the former, `ShardNotAssigned` for the + latter. `apply_shard_assignment_effects` then calls the *other* `on_shard_assignment_changed` + (`durable_host/mod.rs`, the `WorkerCtx` hook) to recover agents on shards newly held, the + opposite direction from relinquish. +- **Oplog epoch fence.** Every indexed-storage oplog write asserts this executor's current shard + epoch inside the storage transaction (`storage/indexed/{postgres,sqlite}.rs`, surfaced through + `services/oplog/primary.rs`); a shard manager mints a new, higher epoch for the new owner when it + takes over, so a write from an executor that has lost the shard is refused rather than written + (`OplogError::Fenced` / `OplogFence`, carrying the asserted and, when known, the actual epoch). + This is what protects an assignment change this executor has not yet heard about, and a revoked + lease it is still trying to renew: `RelinquishReason::Fenced`. Once one write is refused the + fence *latches* — every later append or commit on that oplog is refused too, without a second + round trip to storage — so nothing further is ever written by this executor for that agent. Only + Postgres and the SQLite-backed indexed storages can fence a write this way + (`IndexedStorage::supports_epoch_fencing`); an executor configured with Redis and a real shard + manager refuses to start rather than run unfenced. + +Either way, `Worker::relinquish` (`worker/mod.rs`) stops the agent without writing to its oplog or +status, drops it from this executor's `ActiveAgents`, and hands its invocation waiters a retriable +error (`ShardingNotReady`, or the fenced-specific variant) rather than an in-place restart — the +oplog is left exactly as it was, for the shard's new owner to reconstruct from when the worker +service routes a request there. An invocation still pending in this executor's queue when it +relinquishes is failed the same way: with a retriable error and no cached result, never with a +result the queue happened to already hold, so a client retry runs it exactly once, on the new +owner. See `crash-matrix.md` for the fence's failure modes and `services/active_agents/mod.rs` for +the sweep that relinquishes on an assignment change. + ## Oplog model Entries are positional or hints (`OplogEntry::is_hint()`). Replay consumes positional entries in @@ -525,6 +580,7 @@ satisfies one does not imply the others. | "Cursor reached the end, so I can do the live effect now." | Liveness is `store_is_live(...)`: the primary needs `switch_to_live` to publish after reconstruction fences; an entity Store needs its own `local_live_tail`. Cursor exhaustion is neither. | `pending_replay_to_live_is_fail_closed_until_finished`, `entity_store_liveness_is_scoped_to_its_invocation_mode` | | "Suspension needs a feature-specific safety gate proving the guest is parked." | Arbitrary unload is the baseline; every obligation must be durable or reconstructible. | Simulated-crash tests at arbitrary points (`simulated_crash`, `interrupt`) | | "Restart differs from suspend." | Both discard the `Store` and reconstruct. | `counter_resource_test_2_with_restart` (state continues across an executor restart), `reacquire_permits_restart_preserves_accepted_queued_live_invocation` | +| "Losing a shard reconstructs the agent, like a restart." | It relinquishes instead: stopped here without writing to its oplog or status, dropped, never rebuilt on this executor. Only the new owner reconstructs. | `oplog_fencing_guard_tests` (`lib.rs`), fence tests in `services/oplog/{primary,tests}.rs` | | "A retried RPC attempt executed the target again." | Same key ⇒ same target invocation; count target mutations, not attempts. | Provider-side counter tests in `tests/rpc.rs` | | "Atomic rollback should generate a fresh RPC key." | Logical counter is owned by the outermost atomic region; keys survive `Jump`. | `tests/transactions.rs`, `tests/revert.rs` | | "Equal return values prove deduplication." | Deterministic echoes are equal even with duplicate execution; count side effects. | Counter-based RPC tests | diff --git a/.agents/skills/understanding-durable-execution/reference/crash-matrix.md b/.agents/skills/understanding-durable-execution/reference/crash-matrix.md index 79c0ac4526..eb4a1ef9e3 100644 --- a/.agents/skills/understanding-durable-execution/reference/crash-matrix.md +++ b/.agents/skills/understanding-durable-execution/reference/crash-matrix.md @@ -1,10 +1,16 @@ # Crash-window matrix -"Crash" here means any loss of the resident runtime: process death, `Restart` (simulated crash), -`Suspend`, eviction, resharding (`on_shard_assignment_changed`), or an executor drop in a test. -Reconstruction is identical in every case: new `Store`, `prepare_instance`, `resume_replay`, -publish Live. The matrix says what the next incarnation does for a crash inside each window and -which durable fact makes that safe. +"Crash" here means any loss of the resident runtime *on the same executor*: process death, +`Restart` (simulated crash), `Suspend`, eviction, or an executor drop in a test. Reconstruction is +identical in every case: new `Store`, `prepare_instance`, `resume_replay`, publish Live. The +matrix says what the next incarnation does for a crash inside each window and which durable fact +makes that safe. + +Resharding and the oplog epoch fence are different: this executor does not reconstruct at all. It +relinquishes the agent (`InterruptKind::ShardLost`) — stopped without writing to its oplog or +status, dropped here — and the shard's new owner is the one that runs `prepare_instance` / +`resume_replay`, on its own copy of the same oplog. See "Resharding and the oplog epoch fence" +below for what that leaves behind. ## Durable host call (`concurrent/call.rs`, `concurrent/delivery.rs`) @@ -92,6 +98,23 @@ which durable fact makes that safe. | Body traps | no entity terminal | Owner invocation fails; owner group drains; siblings blocked on the lane are fenced | `guest_trap_fences_a_blocked_sibling_and_drains_the_owner_group` | | Owner reaches replay tail while a body is still reconstructing | — | `HistoricalReconstruction` fences keep `PendingReplayToLive` closed until every active body validates | `completed_reconstruction_claim_blocks_concurrent_replay_to_live` | +## Resharding and the oplog epoch fence (`worker/mod.rs::relinquish`, `services/oplog/primary.rs`) + +Two triggers relinquish an agent instead of reconstructing it here: the shard manager revoking or +reassigning the shard (`grpc/mod.rs::revoke_shards_internal` / `assign_shards_internal`, +`RelinquishReason::ShardRevoked` / `ShardNotAssigned`), and a write refused because the epoch this +executor asserted no longer matches storage (`OplogError::Fenced`, `RelinquishReason::Fenced`). +Only Postgres and the SQLite-backed indexed storages can refuse a write this way; an executor +configured with Redis and a real shard manager refuses to start rather than run unfenced. + +| Crash window | Oplog shape left behind | What happens here | Durable fact relied on | +|---|---|---|---| +| Assignment revoked/reassigned, before any write is attempted | whatever was already committed | `relinquish_matching` stops matching agents directly; no write is attempted or refused | `ShardService::check_worker` / the delivered assignment, not the oplog | +| A write is attempted after the shard actually moved | nothing new; the attempted entry is refused, not partially written | The refusal is returned (`OplogError::Fenced`), not retried or swallowed; the agent relinquishes | Epoch asserted inside the storage transaction | +| Any later write on the same oplog handle | still nothing new | The fence latches: every later add/commit is refused immediately, without a second storage round trip | The oplog's own latched `OplogFence` | +| An invocation still queued when relinquish runs | unaffected | Failed with a retriable error (`ShardingNotReady` / the fenced variant), never a cached result | `PendingLiveInvocationDisposition::Fail` | +| The new owner opens the same agent | the fenced executor's last accepted entries | Ordinary `prepare_instance` / `resume_replay`, from committed history exactly as it was left | Nothing was written after the fence latched | + ## Oplog-processor plugins (`services/oplog/plugin.rs`) | Crash window | Recovery | diff --git a/Makefile.toml b/Makefile.toml index 0a9a0da0c2..308ac542ae 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -725,6 +725,7 @@ cargo-test-r run --package golem-registry-service --test '*' -- --nocapture --re cargo-test-r run --package golem-worker-service --test '*' -- --nocapture --report-time $JUNIT_OPTS RUST_LOG=debug cargo-test-r run --package golem-debugging-service --test '*' -- --report-time $JUNIT_OPTS cargo-test-r run --package golem-shard-manager --test integration -- --nocapture --report-time $JUNIT_OPTS +cargo-test-r run --package golem-test-framework --test signal_unreaped_child -- --nocapture --report-time $JUNIT_OPTS ''' [tasks.integration-tests-group6] diff --git a/golem-debugging-service/src/debug_context.rs b/golem-debugging-service/src/debug_context.rs index 0ea495f414..97954a0bce 100644 --- a/golem-debugging-service/src/debug_context.rs +++ b/golem-debugging-service/src/debug_context.rs @@ -329,7 +329,7 @@ impl UpdateManagement for DebugContext { &self, target_revision: ComponentRevision, details: Option, - ) { + ) -> Result<(), WorkerExecutorError> { self.durable_ctx .on_worker_update_failed(target_revision, details) .await @@ -340,7 +340,7 @@ impl UpdateManagement for DebugContext { target_revision: ComponentRevision, new_component_size: u64, new_active_plugins: HashSet, - ) { + ) -> Result<(), WorkerExecutorError> { self.durable_ctx .on_worker_update_succeeded(target_revision, new_component_size, new_active_plugins) .await diff --git a/golem-shard-manager/src/quota/quota_service_tests.rs b/golem-shard-manager/src/quota/quota_service_tests.rs index e731f2ee74..56bf5c6861 100644 --- a/golem-shard-manager/src/quota/quota_service_tests.rs +++ b/golem-shard-manager/src/quota/quota_service_tests.rs @@ -379,6 +379,31 @@ async fn renew_lease_rejects_stale_epoch() { assert_eq!(l3.epoch(), l2.epoch().next()); } +#[test] +// `epoch` is a client-supplied argument straight off the wire (`grpc.rs` builds it as +// `LeaseEpoch(request.epoch)` with no validation), and `LeaseEpoch::next()` panics on overflow. +// A `u64::MAX` claim must be rejected as an ordinary stale epoch rather than aborting the +// process. +async fn renew_lease_rejects_u64_max_epoch_instead_of_panicking() { + let fetcher = Arc::new(InMemoryFetcher::new()); + let env = env_id(); + let def = make_definition(env, "tokens"); + let id = def.id; + fetcher.put(def).await; + + let svc = QuotaService::new(test_config(), fetcher, test_repo()); + let pod = test_pod(); + + svc.acquire_lease(env, ResourceName("tokens".into()), pod) + .await + .unwrap(); + + let result = svc + .renew_lease(id, pod, LeaseEpoch(u64::MAX), 0, vec![]) + .await; + assert!(matches!(result, Err(QuotaError::StaleEpoch { .. }))); +} + #[test] async fn renew_lease_fails_for_unknown_pod() { let fetcher = Arc::new(InMemoryFetcher::new()); @@ -565,6 +590,27 @@ async fn release_lease_rejects_stale_epoch() { svc.release_lease(id, pod, l2.epoch(), 0).await.unwrap(); } +#[test] +// Same defect class as `renew_lease_rejects_u64_max_epoch_instead_of_panicking`, on the release +// path's own `epoch.next()` comparison. +async fn release_lease_rejects_u64_max_epoch_instead_of_panicking() { + let fetcher = Arc::new(InMemoryFetcher::new()); + let env = env_id(); + let def = make_definition(env, "tokens"); + let id = def.id; + fetcher.put(def).await; + + let svc = QuotaService::new(test_config(), fetcher, test_repo()); + let pod = test_pod(); + + svc.acquire_lease(env, ResourceName("tokens".into()), pod) + .await + .unwrap(); + + let result = svc.release_lease(id, pod, LeaseEpoch(u64::MAX), 0).await; + assert!(matches!(result, Err(QuotaError::StaleEpoch { .. }))); +} + #[test] async fn release_lease_allows_re_acquire() { let fetcher = Arc::new(InMemoryFetcher::new()); diff --git a/golem-shard-manager/src/quota/quota_state.rs b/golem-shard-manager/src/quota/quota_state.rs index 66a055d855..25039e24a7 100644 --- a/golem-shard-manager/src/quota/quota_state.rs +++ b/golem-shard-manager/src/quota/quota_state.rs @@ -24,6 +24,19 @@ use std::collections::HashMap; use std::time::Duration; use tracing::debug; +/// Whether `epoch` immediately precedes `next`, without calling `LeaseEpoch::next()` on `epoch` +/// itself - which panics at `u64::MAX` (`checked_add(1).expect(..)`). +/// +/// `epoch` here is the caller's claimed epoch, taken straight off the wire (a `renew_lease` / +/// `release_lease` argument, itself `golem_common::model::quota::LeaseEpoch(request.epoch)` in +/// `grpc.rs` with nothing upstream bounding it) - unlike `pod_lease.epoch`, which only ever +/// advances by exactly one through this state's own `next()` calls. A `u64::MAX` claim can never +/// legitimately precede a real stored epoch, so it simply fails this check like any other stale +/// one, rather than aborting the process. +fn precedes(epoch: LeaseEpoch, next: LeaseEpoch) -> bool { + epoch.0.checked_add(1) == Some(next.0) +} + pub(super) struct AcquireLeaseResult { pub epoch: LeaseEpoch, pub allocated_amount: u64, @@ -367,7 +380,7 @@ impl QuotaState { let pod_lease = self.leases.get_mut(pod).ok_or(QuotaError::LeaseNotFound { resource_definition_id: self.definition.id, })?; - if epoch.next() != pod_lease.epoch { + if !precedes(epoch, pod_lease.epoch) { return Err(QuotaError::StaleEpoch { resource_definition_id: self.definition.id, provided: epoch, @@ -424,7 +437,7 @@ impl QuotaState { let pod_lease = self.leases.get(pod).ok_or(QuotaError::LeaseNotFound { resource_definition_id: self.definition.id, })?; - if epoch.next() != pod_lease.epoch { + if !precedes(epoch, pod_lease.epoch) { return Err(QuotaError::StaleEpoch { resource_definition_id: self.definition.id, provided: epoch, diff --git a/golem-shard-manager/src/sharding/model.rs b/golem-shard-manager/src/sharding/model.rs index 191236ac65..8fd9915aaf 100644 --- a/golem-shard-manager/src/sharding/model.rs +++ b/golem-shard-manager/src/sharding/model.rs @@ -36,7 +36,15 @@ impl ShardEpoch { } pub fn next(self) -> Self { - Self(self.0.checked_add(1).expect("ShardEpoch overflow")) + self.checked_next().expect("ShardEpoch overflow") + } + + /// `None` at `u64::MAX`, rather than panicking. Callers deriving an epoch to *store* from + /// untrusted input (a wire claim or fenced report - see `raise_epoch_floor_for`) use this: the + /// value that ends up in `shard_epochs` must never be `u64::MAX` itself, or the next ordinary + /// reassignment's call to [`Self::next`] on it panics instead of this one. + pub fn checked_next(self) -> Option { + self.0.checked_add(1).map(Self) } } @@ -572,9 +580,30 @@ impl ShardLeaseState { // clears the claimant's oplog rows. An unassigned shard has no entry to corrupt, and // raising its high-water only makes the next mint start above the epoch the claim // proves is already in use. - let epoch = match self.shard_assignments.get(shard_id) { - Some(entry) if Some(entry.executor_id) != holder => claimed_epoch.next(), - _ => *claimed_epoch, + let mints_past_claim = self + .shard_assignments + .get(shard_id) + .is_some_and(|entry| Some(entry.executor_id) != holder); + // The guard is on the *candidate* - what this would actually store - not on + // `claimed_epoch` itself: a claim of `u64::MAX` overflows `checked_next` right here, + // but a claim of `u64::MAX - 1` does not, and minting past it succeeds, landing + // exactly on `u64::MAX`. Either way the wire carries a raw `u64` with nothing + // upstream bounding it, and a `u64::MAX` epoch must never reach `shard_epochs`: the + // next ordinary reassignment mints past whatever is stored there + // (`next_epoch_for`), and `ShardEpoch::next` panics on it - aborting this process, + // and again on every retry of the same report. So a candidate that would land on it + // is dropped here, the same stance as the out-of-range shard id above. + let candidate = if mints_past_claim { + claimed_epoch.checked_next() + } else { + Some(*claimed_epoch) + }; + let Some(epoch) = candidate.filter(|epoch| epoch.0 != u64::MAX) else { + warn!( + shard_id = %shard_id, + "Ignoring an out-of-range shard epoch; storing it would overflow a later mint" + ); + continue; }; self.shard_epochs.insert(*shard_id, epoch); if let Some(entry) = self.shard_assignments.get_mut(shard_id) { @@ -1344,6 +1373,117 @@ mod tests { ); } + #[test] + // The wire carries a raw u64 with nothing upstream bounding it - a corrupted report, or a + // storage bug that turns a negative epoch into one near u64::MAX on read-back, must not reach + // `ShardEpoch::next`, which panics on overflow and would abort the process on every retry of + // the same report. Covers both funnels: a claim (`raise_epoch_floor`) and a fenced report + // (`raise_epoch_floor_past`). + fn an_out_of_range_epoch_is_ignored_rather_than_overflowing() { + let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[1])]); + + // A claim on another executor's shard would normally re-mint its owner one past it; at + // u64::MAX that mint is exactly the overflow this must avoid. + let claim_at_max = BTreeMap::from([(shard(0), ShardEpoch(u64::MAX))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &claim_at_max) + .is_empty(), + "a claim at u64::MAX must be ignored, not minted past" + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); + assert_eq!( + shard_state.shard_epochs.get(&shard(0)), + Some(&ShardEpoch(0)), + "the out-of-range claim must not even be recorded as a high-water mark" + ); + assert!(shard_state.check_invariants().is_ok()); + + // Same for a fenced report, whose default holder is `None` - so a claim naming an entry + // the reporter itself owns takes the same path here as `raise_epoch_floor`'s "another + // executor" branch. + let fenced_at_max = BTreeMap::from([(shard(1), ShardEpoch(u64::MAX))]); + assert!( + shard_state + .raise_epoch_floor_past(&fenced_at_max) + .is_empty(), + "a fenced epoch at u64::MAX must be ignored, not minted past" + ); + assert_eq!(shard_state.epoch_for_shard(shard(1)), Some(ShardEpoch(0))); + assert!(shard_state.check_invariants().is_ok()); + + // An unassigned shard would otherwise store the claim verbatim as its high-water mark; + // that path must refuse it too, since a later assignment would mint past the stored value. + let unassigned_at_max = BTreeMap::from([(shard(2), ShardEpoch(u64::MAX))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &unassigned_at_max) + .is_empty() + ); + assert!(!shard_state.shard_epochs.contains_key(&shard(2))); + + // The narrower defect: `u64::MAX - 1` does NOT overflow `checked_next` by itself, so a + // guard on the raw claim (as an earlier version of this fix had) lets it through. On + // another executor's shard it reaches the mint-one-past branch, mints successfully, and + // lands exactly on `u64::MAX` - which must still be refused, because the guard has to be + // on what would be *stored*, not on the input. + let claim_near_max = BTreeMap::from([(shard(0), ShardEpoch(u64::MAX - 1))]); + assert!( + shard_state + .raise_epoch_floor(executor(2), &claim_near_max) + .is_empty(), + "a claim of u64::MAX - 1 on another executor's shard must be ignored: minting past \ + it would store exactly u64::MAX" + ); + assert_eq!( + shard_state.epoch_for_shard(shard(0)), + Some(ShardEpoch(0)), + "the rejected claim must not have moved the epoch at all" + ); + assert!(shard_state.check_invariants().is_ok()); + + // Proof this actually prevents the crash, not just that the claim was reported as + // rejected: an ordinary reassignment of the same shard afterwards must not panic. Before + // the candidate-based guard, the claim above would have stored `u64::MAX` on shard 0, and + // `next_epoch_for` (via `assign_shard`, below) would have called `ShardEpoch::next` on it + // and panicked. + shard_state.remove_executor(executor(1)); + shard_state.add_executor(executor(3), addr(3), None, t0(), TTL); + assert_eq!( + shard_state.assign_shard(executor(3), shard(0)), + ShardEpoch(1), + "an ordinary reassignment after the rejected claim mints normally, one past the \ + epoch that was never disturbed" + ); + assert!(shard_state.check_invariants().is_ok()); + + // Same class, on the fenced-report funnel (holder `None`, so any assigned entry takes + // the mint-one-past branch). + let fenced_near_max = BTreeMap::from([(shard(1), ShardEpoch(u64::MAX - 1))]); + assert!( + shard_state + .raise_epoch_floor_past(&fenced_near_max) + .is_empty(), + "a fenced epoch of u64::MAX - 1 must be ignored for the same reason" + ); + assert_eq!(shard_state.epoch_for_shard(shard(1)), Some(ShardEpoch(0))); + + // Contrast with the unassigned/verbatim branch: storing `u64::MAX - 1` there calls + // `.next()` on nothing, so nothing overflows yet, and it is legitimately accepted - + // unlike literal `u64::MAX` above, which is refused in every branch. + let unassigned_near_max = BTreeMap::from([(shard(2), ShardEpoch(u64::MAX - 1))]); + assert_eq!( + shard_state.raise_epoch_floor(executor(2), &unassigned_near_max), + vec![shard(2)], + "u64::MAX - 1 does not overflow anything when stored verbatim, so it is accepted" + ); + assert_eq!( + shard_state.shard_epochs.get(&shard(2)), + Some(&ShardEpoch(u64::MAX - 1)) + ); + assert!(shard_state.check_invariants().is_ok()); + } + #[test] fn a_claim_ahead_on_another_executors_shard_re_mints_its_owner_above_it() { // Executor 1 holds shard 0 at epoch 0 in a state that lost history; executor 2 held it at diff --git a/golem-test-framework/Cargo.toml b/golem-test-framework/Cargo.toml index 450959f9b7..e8d1bb861f 100644 --- a/golem-test-framework/Cargo.toml +++ b/golem-test-framework/Cargo.toml @@ -12,6 +12,11 @@ license-file = "../LICENSE" [lib] harness = false +[[test]] +name = "signal_unreaped_child" +path = "tests/signal_unreaped_child.rs" +harness = false + [dependencies] golem-api-grpc = { workspace = true } golem-client = { workspace = true } diff --git a/golem-test-framework/src/components/worker_executor/spawned.rs b/golem-test-framework/src/components/worker_executor/spawned.rs index d05eb30ec3..a170e25b29 100644 --- a/golem-test-framework/src/components/worker_executor/spawned.rs +++ b/golem-test-framework/src/components/worker_executor/spawned.rs @@ -211,8 +211,11 @@ impl SpawnedWorkerExecutor { /// A reaped child's pid is free for the OS to hand to an unrelated process, and `is_running`'s /// `try_wait` reaps an exited child while leaving it in place, so a raw `kill` on `child.id()` /// alone could signal a stranger. `Child::kill` has this guard built in; `kill(2)` does not. +/// +/// `pub` so its process-supervision behavior can be pinned by an integration test under +/// `golem-test-framework/tests/` rather than a `--lib` unit test that would spawn a process. #[cfg(unix)] -fn signal_unreaped_child(child: &mut Child, signal: libc::c_int, what: &str) { +pub fn signal_unreaped_child(child: &mut Child, signal: libc::c_int, what: &str) { match child.try_wait() { Ok(None) => {} Ok(Some(status)) => panic!("Cannot {what}: it has already exited ({status})"), @@ -340,47 +343,7 @@ impl Drop for SpawnedWorkerExecutor { } } -#[cfg(all(test, unix))] -mod tests { - use test_r::test; - - use super::signal_unreaped_child; - use std::process::{Child, Command}; - - /// Kills and reaps the child when the test ends, also when an assertion panics, so a failing - /// run does not leave a stopped process behind. - struct KilledOnDrop(Child); - - impl Drop for KilledOnDrop { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } - } - - #[test] - #[should_panic(expected = "has already exited")] - fn a_child_that_has_been_reaped_is_never_signalled() { - let mut child = Command::new("true") - .spawn() - .expect("failed to spawn `true`"); - // Records the exit status, exactly as `is_running`'s `try_wait` does for an exited child. - child.wait().expect("failed to wait for `true`"); - - signal_unreaped_child(&mut child, libc::SIGSTOP, "pause `true`"); - } - - #[test] - fn a_live_child_can_be_stopped_and_continued() { - let mut child = KilledOnDrop( - Command::new("sleep") - .arg("30") - .spawn() - .expect("failed to spawn `sleep`"), - ); - - signal_unreaped_child(&mut child.0, libc::SIGSTOP, "pause `sleep`"); - // `try_wait` does not report a stopped child, so continuing it is not refused as exited. - signal_unreaped_child(&mut child.0, libc::SIGCONT, "resume `sleep`"); - } -} +// `signal_unreaped_child`'s process-supervision behavior is pinned by +// `golem-test-framework/tests/signal_unreaped_child.rs` instead of a `--lib` unit test: unit +// tests must never spawn external processes (AGENTS.md), and `cargo make unit-tests` runs +// `--workspace --lib`. diff --git a/golem-test-framework/tests/signal_unreaped_child.rs b/golem-test-framework/tests/signal_unreaped_child.rs new file mode 100644 index 0000000000..39805501cf --- /dev/null +++ b/golem-test-framework/tests/signal_unreaped_child.rs @@ -0,0 +1,64 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pins `signal_unreaped_child`'s process-supervision behavior. It lives here rather than as a +//! `--lib` unit test in `spawned.rs` because it needs a real child process to signal, and unit +//! tests must never spawn external processes (AGENTS.md); `cargo make unit-tests` runs +//! `--workspace --lib` and would otherwise pick it up. + +test_r::enable!(); + +#[cfg(unix)] +mod unix { + use golem_test_framework::components::worker_executor::spawned::signal_unreaped_child; + use std::process::{Child, Command}; + use test_r::test; + + /// Kills and reaps the child when the test ends, also when an assertion panics, so a failing + /// run does not leave a stopped process behind. + struct KilledOnDrop(Child); + + impl Drop for KilledOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[test] + #[should_panic(expected = "has already exited")] + fn a_child_that_has_been_reaped_is_never_signalled() { + let mut child = Command::new("true") + .spawn() + .expect("failed to spawn `true`"); + // Records the exit status, exactly as `is_running`'s `try_wait` does for an exited child. + child.wait().expect("failed to wait for `true`"); + + signal_unreaped_child(&mut child, libc::SIGSTOP, "pause `true`"); + } + + #[test] + fn a_live_child_can_be_stopped_and_continued() { + let mut child = KilledOnDrop( + Command::new("sleep") + .arg("30") + .spawn() + .expect("failed to spawn `sleep`"), + ); + + signal_unreaped_child(&mut child.0, libc::SIGSTOP, "pause `sleep`"); + // `try_wait` does not report a stopped child, so continuing it is not refused as exited. + signal_unreaped_child(&mut child.0, libc::SIGCONT, "resume `sleep`"); + } +} diff --git a/golem-worker-executor-test-utils/src/lib.rs b/golem-worker-executor-test-utils/src/lib.rs index 8551cd4a07..1fa8d44474 100644 --- a/golem-worker-executor-test-utils/src/lib.rs +++ b/golem-worker-executor-test-utils/src/lib.rs @@ -794,7 +794,7 @@ impl TestWorkerExecutor { .try_get_worker(&owned_agent_id) .await .ok_or_else(|| anyhow!("worker is not loaded: {owned_agent_id}"))?; - worker.queue_card_revocation(card_id).await; + worker.queue_card_revocation(card_id).await?; Ok(()) } @@ -833,7 +833,7 @@ impl TestWorkerExecutor { None, golem_common::base_model::oplog::QueuedCardEvent::install(card), )) - .await; + .await?; Ok(()) } @@ -2253,7 +2253,7 @@ impl UpdateManagement for TestWorkerCtx { &self, target_revision: ComponentRevision, details: Option, - ) { + ) -> Result<(), WorkerExecutorError> { self.durable_ctx .on_worker_update_failed(target_revision, details) .await @@ -2264,7 +2264,7 @@ impl UpdateManagement for TestWorkerCtx { target_revision: ComponentRevision, new_component_size: u64, new_active_plugins: HashSet, - ) { + ) -> Result<(), WorkerExecutorError> { self.durable_ctx .on_worker_update_succeeded(target_revision, new_component_size, new_active_plugins) .await diff --git a/golem-worker-executor/src/durable_host/call_coordinator.rs b/golem-worker-executor/src/durable_host/call_coordinator.rs index fcef987dcd..9ebb590b06 100644 --- a/golem-worker-executor/src/durable_host/call_coordinator.rs +++ b/golem-worker-executor/src/durable_host/call_coordinator.rs @@ -142,7 +142,7 @@ impl<'a, Ctx: WorkerCtx> DurableCallCoordinator<'a, Ctx> { .public_state .worker() .commit_oplog_and_update_state(CommitLevel::DurableOnly) - .await; + .await?; // The status checkpoint is only safe after the durable boundary has committed. self.ctx.maybe_mid_invocation_checkpoint().await; } @@ -656,7 +656,7 @@ where reason, ), }; - worker.add_and_commit_oplog(entry).await; + worker.add_and_commit_oplog(entry).await?; Ok(()) } @@ -702,7 +702,7 @@ where reason, ), }; - worker.add_and_commit_oplog(entry).await; + worker.add_and_commit_oplog(entry).await?; Ok(()) } @@ -764,7 +764,7 @@ where card_id, Some(wallet_generation), )) - .await; + .await?; } Ok(()) } @@ -1009,7 +1009,7 @@ where } worker .queue_card_revocations_locked(&revoked_card_ids) - .await; + .await?; Ok(()) } @@ -1140,7 +1140,7 @@ where target_holder, store.with(|mut access| Some(get_ctx(access.data_mut()).state.wallet_generation)), )) - .await; + .await?; Ok(()) } @@ -1222,7 +1222,7 @@ where retry.installed_card.card_id(), target_holder, )) - .await; + .await?; Ok(()) } @@ -1291,7 +1291,7 @@ where affected_wallets, local_wallet_generation: Some(wallet_generation), }) - .await; + .await?; Ok(()) } @@ -1528,7 +1528,7 @@ where target_revision, Some(details.clone()), )) - .await; + .await?; tracing::warn!( "Worker failed to update to {}: {}, update attempt aborted", target_revision, @@ -1564,6 +1564,6 @@ where component_size, active_plugins, ) - .await; + .await?; Ok(()) } diff --git a/golem-worker-executor/src/durable_host/clocks/monotonic_clock.rs b/golem-worker-executor/src/durable_host/clocks/monotonic_clock.rs index 4fe3dfc653..343d149688 100644 --- a/golem-worker-executor/src/durable_host/clocks/monotonic_clock.rs +++ b/golem-worker-executor/src/durable_host/clocks/monotonic_clock.rs @@ -138,7 +138,7 @@ impl HostWithStore for HasSelf InFunctionRetryHost for ScopedRetryHo retry_from: OplogIndex, inside_atomic_region: bool, retry_policy_state: Option, - ) { + ) -> Result<(), crate::services::oplog::OplogError> { self.inner .append_retry_error_entry(retry_from, inside_atomic_region, retry_policy_state) - .await; + .await } } @@ -2420,6 +2420,8 @@ impl DurableCallSession { start: begin_index.next(), end: pending.replay_target().next(), }; + // Refused, the scope must not re-run live: its first attempt would be + // replayed by the shard's new owner with no `Jump` skipping it. prepared .public_state .worker() @@ -2427,7 +2429,15 @@ impl DurableCallSession { prepared.entity_parent_start_index, deleted_region, )) - .await; + .await + .map_err(|error| { + ( + WorkerExecutorError::from(error), + AccessStartCleanup { + atomic_lease: prepared.atomic_lease.clone(), + }, + ) + })?; prepared .public_state .worker() @@ -2497,7 +2507,7 @@ impl DurableCallSession { prepared .public_state .worker() - .add_and_commit_oplog_or_fenced(OplogEntry::Start { + .add_and_commit_oplog(OplogEntry::Start { timestamp: Timestamp::now_utc(), parent_start_index: prepared.entity_parent_start_index, function_name: scope_name, @@ -4842,7 +4852,7 @@ where public_state .worker() .commit_oplog_and_update_state(CommitLevel::DurableOnly) - .await; + .await?; if let Some(min_exposed_marker) = store.with(|mut access| { let ctx = get_ctx(access.data_mut()); if ctx.state.at_clean_checkpoint_boundary() { diff --git a/golem-worker-executor/src/durable_host/concurrent/tests.rs b/golem-worker-executor/src/durable_host/concurrent/tests.rs index cd18239698..6d3ffd8e39 100644 --- a/golem-worker-executor/src/durable_host/concurrent/tests.rs +++ b/golem-worker-executor/src/durable_host/concurrent/tests.rs @@ -1658,10 +1658,11 @@ impl InFunctionRetryHost for RetryHostProbe { retry_from: OplogIndex, inside_atomic_region: bool, _retry_policy_state: Option, - ) { + ) -> Result<(), crate::services::oplog::OplogError> { self.appended_retry_from.push(retry_from); self.appended_inside_atomic_region .push(inside_atomic_region); + Ok(()) } } diff --git a/golem-worker-executor/src/durable_host/durability.rs b/golem-worker-executor/src/durable_host/durability.rs index c83ea165bb..151397b2ac 100644 --- a/golem-worker-executor/src/durable_host/durability.rs +++ b/golem-worker-executor/src/durable_host/durability.rs @@ -24,7 +24,7 @@ use crate::metrics::wasm::{ use crate::model::ExecutionStatus; use crate::preview2::golem::durability::durability; use crate::services::environment_state::EnvironmentStateService; -use crate::services::oplog::OplogOps; +use crate::services::oplog::{OplogError, OplogOps}; use crate::services::{HasOplog, HasWorker}; use crate::workerctx::WorkerCtx; use anyhow::Error; @@ -673,12 +673,14 @@ pub trait InFunctionRetryHost { } /// Writes an `OplogEntry::Error` entry for an in-function retry attempt, and commits. + /// + /// A refusal is returned: the retry must not run again for an attempt that was never recorded. async fn append_retry_error_entry( &mut self, retry_from: OplogIndex, inside_atomic_region: bool, retry_policy_state: Option, - ); + ) -> Result<(), OplogError>; } pub(crate) fn collect_named_retry_policies( @@ -960,8 +962,15 @@ impl InFunctionRetryState { } let inside_atomic_region = ctx.retry_context_atomic_region_had_side_effects(); - ctx.append_retry_error_entry(retry_point, inside_atomic_region, retry_policy_state) - .await; + // Refused, the shard has a new owner: the failure is returned instead of retried, and the + // trap it becomes is classified as the lost shard. + if ctx + .append_retry_error_entry(retry_point, inside_atomic_region, retry_policy_state) + .await + .is_err() + { + return AsyncRetryDecision::FallBackToTrap; + } self.retry_count += 1; debug!( @@ -1491,7 +1500,7 @@ impl durability::HostLiveCustomDurableInvocat response: Some(response), forced_commit, }) - .await; + .await?; let checkpoint = accessor.with(|mut access| { let ctx = access.get(); ctx.state.active_custom_invocations.remove(&start_index); @@ -1651,19 +1660,20 @@ impl durability::HostWithStore .upload_payload_owned(request) .await .map_err(|err| format!("Failed to store durable function request: {err}"))?; - Ok::<_, String>( - worker - .add_and_commit_oplog(OplogEntry::Start { - timestamp: Timestamp::now_utc(), - parent_start_index, - function_name, - invocation_id: Some(start_invocation_id), - observational_owner: None, - request: Some(persisted_request), - durable_function_type: start_function_type, - }) - .await, - ) + // A refused `Start` fails the begin: the guest must not perform a side effect the + // shard's new owner, finding no `Start`, would perform again. + worker + .add_and_commit_oplog(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index, + function_name, + invocation_id: Some(start_invocation_id), + observational_owner: None, + request: Some(persisted_request), + durable_function_type: start_function_type, + }) + .await + .map_err(|err| format!("Failed to record durable function start: {err}")) }); let cancellation_worker = accessor.with(|mut access| access.get().public_state.worker()); @@ -1684,7 +1694,7 @@ impl durability::HostWithStore start_index, partial: None, }) - .await; + .await?; Ok(Some(start_index)) } (_, Err(err)) if matches!(verdict, CustomBeginVerdict::Cancelled) => { @@ -1856,9 +1866,9 @@ impl InFunctionRetryHost for DurableWorkerCtx { retry_from: OplogIndex, inside_atomic_region: bool, retry_policy_state: Option, - ) { + ) -> Result<(), OplogError> { if self.state.durability_is_suppressed() { - return; + return Ok(()); } use golem_common::model::oplog::AgentError; @@ -1870,7 +1880,11 @@ impl InFunctionRetryHost for DurableWorkerCtx { inside_atomic_region, retry_policy_state, ); - self.public_state.worker().add_and_commit_oplog(entry).await; + self.public_state + .worker() + .add_and_commit_oplog(entry) + .await?; + Ok(()) } } @@ -2376,7 +2390,7 @@ impl InFunctionRetryHost for TaskRetryContext { retry_from: OplogIndex, inside_atomic_region: bool, retry_policy_state: Option, - ) { + ) -> Result<(), OplogError> { use golem_common::model::oplog::AgentError; let entry = OplogEntry::error( self.entity_parent_start_index, @@ -2386,9 +2400,10 @@ impl InFunctionRetryHost for TaskRetryContext { inside_atomic_region, retry_policy_state.clone(), ); - self.worker.add_and_commit_oplog(entry).await; + self.worker.add_and_commit_oplog(entry).await?; self.current_retry_policy_state = retry_policy_state; + Ok(()) } } @@ -2685,9 +2700,10 @@ mod tests { _retry_from: OplogIndex, _inside_atomic_region: bool, retry_policy_state: Option, - ) { + ) -> Result<(), OplogError> { self.retry_entries_appended += 1; self.current_retry_policy_state = retry_policy_state; + Ok(()) } } diff --git a/golem-worker-executor/src/durable_host/durable_session.rs b/golem-worker-executor/src/durable_host/durable_session.rs index c9fab18c32..3774457620 100644 --- a/golem-worker-executor/src/durable_host/durable_session.rs +++ b/golem-worker-executor/src/durable_host/durable_session.rs @@ -968,7 +968,7 @@ impl DurableSessionStreams { epoch, }, )) - .await; + .await?; self.commit_consumer_journal().await?; Ok(true) } @@ -1057,11 +1057,18 @@ impl DurableSessionStreams { } } - pub(crate) async fn append_record(&self, record: StreamSessionRecordV1) { - self.producer + pub(crate) async fn append_record(&self, record: StreamSessionRecordV1) -> Result<(), String> { + match self + .producer .append_session_record_attributed(self.entity_parent_start_index, record) .await - .expect("internally generated durable session record is valid"); + { + Ok(()) => Ok(()), + Err(error @ DurableStreamProducerError::Fenced(_)) => Err(error.to_string()), + Err(error) => { + panic!("internally generated durable session record is invalid: {error}") + } + } } async fn try_append_record(&self, record: StreamSessionRecordV1) -> Result<(), String> { @@ -1091,7 +1098,7 @@ impl DurableSessionStreams { attempt_id, }, )) - .await; + .await?; self.commit_consumer_journal().await?; Ok(attempt_id) } @@ -1202,7 +1209,7 @@ impl DurableSessionStreams { mapping, }, )) - .await; + .await?; Ok(()) } @@ -2099,7 +2106,7 @@ impl DurableSessionStreams { Some(existing) => existing, None => { self.append_record(StreamSessionRecordV1::ConsumerCancelIntent(intent.clone())) - .await; + .await?; self.commit_consumer_journal().await?; intent } @@ -2721,7 +2728,7 @@ impl DurableSessionStreams { } } else { self.append_record(StreamSessionRecordV1::InvocationResult(record)) - .await; + .await?; self.commit_consumer_journal().await?; } self.decode_initial( @@ -4688,7 +4695,7 @@ impl DurableInputProducer { }), }; if !journaled { - streams.append_record(record).await; + streams.append_record(record).await?; streams.commit_consumer_journal().await?; let committed_through = queued_events .back() @@ -7135,7 +7142,7 @@ mod tests { ), ] { assert!(record.has_supported_format()); - streams.append_record(record).await; + streams.append_record(record).await.unwrap(); } producer .prepare_attachment(attachment.clone(), 100) @@ -7166,7 +7173,8 @@ mod tests { terminal: StreamConsumerTerminalV1::End(StreamEndResultV1::Ok), }, )) - .await; + .await + .unwrap(); streams.commit_consumer_journal().await.unwrap(); producer.finalize_attachment(attachment.clone(), golem_common::model::durable_stream::StreamAttachmentFinalizationReasonV1::ConsumerFinalized, 101).await.unwrap(); let mut next_attachment = attachment; @@ -7272,7 +7280,8 @@ mod tests { consumer_read_ordinal: 0, }, )) - .await; + .await + .unwrap(); let endpoint = streams .endpoint(handle, 0, SessionStreamRoleV1::Input) @@ -7975,7 +7984,8 @@ mod tests { stream_mappings: vec![mapping.clone()], }, )) - .await; + .await + .unwrap(); let pending_invocation_oplog_index = consumer_oplog .add(OplogEntry::pending_agent_invocation( consumer.invocation.idempotency_key.clone(), @@ -7997,7 +8007,8 @@ mod tests { pending_invocation_oplog_index, }, )) - .await; + .await + .unwrap(); let streams = streams.with_attachment(1, attempt_id); remote_producer .prepare_attachment(attachment.clone(), 100) @@ -8321,7 +8332,8 @@ mod tests { mapping: partial_mapping.clone(), }, )) - .await; + .await + .unwrap(); let consumer_length = restarted.oplog.current_oplog_index().await; let producer_length = producer_oplog.current_oplog_index().await; assert!( @@ -8348,7 +8360,8 @@ mod tests { mapping: partial_mapping, }, )) - .await; + .await + .unwrap(); assert!(restarted.complete().await.is_err()); } @@ -8830,7 +8843,8 @@ mod tests { pending_invocation_oplog_index, }, )) - .await; + .await + .unwrap(); let streams = streams.with_attachment(1, attempt_id); assert!( streams @@ -9186,7 +9200,8 @@ mod tests { stream_mappings: Vec::new(), }, )) - .await; + .await + .unwrap(); let pending_invocation_oplog_index = consumer_oplog .add(OplogEntry::pending_agent_invocation( consumer.invocation.idempotency_key.clone(), @@ -9208,7 +9223,8 @@ mod tests { pending_invocation_oplog_index, }, )) - .await; + .await + .unwrap(); let epoch1 = streams.with_attachment(1, start_attempt_id); let root_mapping = StreamSessionMappingRecordV1 { transport_stream_id: 17, diff --git a/golem-worker-executor/src/durable_host/golem/retry_api.rs b/golem-worker-executor/src/durable_host/golem/retry_api.rs index 1d3844fcf0..fb5d4a4e6a 100644 --- a/golem-worker-executor/src/durable_host/golem/retry_api.rs +++ b/golem-worker-executor/src/durable_host/golem/retry_api.rs @@ -147,7 +147,7 @@ impl Host for DurableWorkerCtx { self.entity_parent_start_index(), named_policy.clone(), )) - .await; + .await?; } self.state.apply_set_retry_policy(named_policy); @@ -176,7 +176,7 @@ impl Host for DurableWorkerCtx { self.entity_parent_start_index(), name.clone(), )) - .await; + .await?; } self.state.apply_remove_retry_policy(&name); diff --git a/golem-worker-executor/src/durable_host/golem/v1x.rs b/golem-worker-executor/src/durable_host/golem/v1x.rs index 7ffc12f8c1..61d39b0128 100644 --- a/golem-worker-executor/src/durable_host/golem/v1x.rs +++ b/golem-worker-executor/src/durable_host/golem/v1x.rs @@ -717,7 +717,7 @@ impl Host for DurableWorkerCtx { self.public_state .worker() .add_and_commit_oplog(OplogEntry::jump(self.entity_parent_start_index(), jump)) - .await; + .await?; debug!("Interrupting live execution for jumping from {jump_source} to {jump_target}",); Err(InterruptKind::Jump.into()) @@ -820,7 +820,7 @@ impl Host for DurableWorkerCtx { self.entity_parent_start_index(), deleted_region, )) - .await; + .await?; // TODO: this recomputation should not be necessary. self.public_state.worker().reattach_worker_status().await; @@ -1613,7 +1613,7 @@ impl Host for DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; let created_by = self.created_by(); let fork_result = loop { diff --git a/golem-worker-executor/src/durable_host/http/types.rs b/golem-worker-executor/src/durable_host/http/types.rs index 7ce9ee8a9e..cd6fcdebd4 100644 --- a/golem-worker-executor/src/durable_host/http/types.rs +++ b/golem-worker-executor/src/durable_host/http/types.rs @@ -21,7 +21,7 @@ use crate::durable_host::http::inline_retry::{ use crate::durable_host::http::{continue_http_request, end_http_request}; use crate::durable_host::{DurabilityHost, DurableWorkerCtx}; use crate::services::HasWorker; -use crate::services::oplog::{CommitLevel, OplogOps}; +use crate::services::oplog::{CommitLevel, OplogError, OplogOps}; use crate::workerctx::WorkerCtx; use golem_common::model::NamedRetryPolicy; use golem_common::model::oplog::host_functions::{ @@ -1161,7 +1161,7 @@ impl HostFutureIncomingResponse for DurableWorkerCtx { _ => None, }; } - persist_http_response(self, request, &serializable_response, begin_index).await; + persist_http_response(self, request, &serializable_response, begin_index).await?; if !is_pending && let Ok(Some(Ok(Ok(resource)))) = &response { let incoming_response_handle = resource.rep(); @@ -1422,7 +1422,7 @@ async fn persist_http_response( request: golem_common::model::oplog::HostRequestHttpRequest, serializable_response: &SerializableHttpResponse, begin_index: golem_common::model::oplog::OplogIndex, -) { +) -> Result<(), WorkerExecutorError> { if !ctx.state.durability_is_suppressed() { ctx.state .oplog @@ -1436,12 +1436,16 @@ async fn persist_http_response( Some(begin_index), ) .await - .unwrap_or_else(|err| panic!("failed to serialize http response: {err}")); + .map_err(|err| match err { + OplogError::Fenced(_) => err, + err => panic!("failed to serialize http response: {err}"), + })?; ctx.public_state .worker() .commit_oplog_and_update_state(CommitLevel::DurableOnly) - .await; + .await?; } + Ok(()) } /// Typed HTTP failure for retry classification, preserving the original `ErrorCode` diff --git a/golem-worker-executor/src/durable_host/mod.rs b/golem-worker-executor/src/durable_host/mod.rs index 122f4ab93c..69f7d52977 100644 --- a/golem-worker-executor/src/durable_host/mod.rs +++ b/golem-worker-executor/src/durable_host/mod.rs @@ -86,7 +86,7 @@ use crate::services::key_value::KeyValueService; use crate::services::linear_memory::{ LinearMemoryTracker, SHARED_LINEAR_MEMORY_ERROR, UnsharedMemoryGrowth, }; -use crate::services::oplog::{CommitLevel, Oplog, OplogOps, OplogService}; +use crate::services::oplog::{CommitLevel, Oplog, OplogError, OplogOps, OplogService}; use crate::services::promise::PromiseService; use crate::services::quota::QuotaService; use crate::services::rdbms::RdbmsService; @@ -647,7 +647,8 @@ impl DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await + .expect("test oplog commit was refused"); } pub(crate) fn entity_reconstruction_claim_hook( @@ -2074,7 +2075,7 @@ impl DurableWorkerCtx { card_id, reason, )) - .await; + .await?; } Ok(Err(reason)) } else { @@ -2086,7 +2087,7 @@ impl DurableWorkerCtx { card, Some(self.state.wallet_generation), )) - .await; + .await?; Ok(Ok(())) } } @@ -2109,7 +2110,7 @@ impl DurableWorkerCtx { card_id, reason, )) - .await; + .await?; return Ok(Err(reason)); } @@ -2126,7 +2127,7 @@ impl DurableWorkerCtx { card, Some(self.state.wallet_generation), )) - .await; + .await?; Ok(Ok(())) } @@ -2158,7 +2159,7 @@ impl DurableWorkerCtx { card_id, Some(self.state.wallet_generation), )) - .await; + .await?; } Ok(()) @@ -2204,7 +2205,10 @@ impl DurableWorkerCtx { local_wallet_generation: Some(self.state.wallet_generation), }; if commit_immediately { - self.public_state.worker().add_and_commit_oplog(entry).await; + self.public_state + .worker() + .add_and_commit_oplog(entry) + .await?; } else { self.public_state.worker().add_to_oplog(entry).await?; } @@ -2245,7 +2249,7 @@ impl DurableWorkerCtx { card_id, Some(wallet_generation), )) - .await; + .await?; } Ok(()) } @@ -2936,7 +2940,7 @@ impl DurableWorkerCtx { let begin_index = self .public_state .worker() - .add_and_commit_oplog_or_fenced(entry) + .add_and_commit_oplog(entry) .await .map_err(WorkerExecutorError::from)?; Ok(begin_index) @@ -3029,7 +3033,7 @@ impl DurableWorkerCtx { self.entity_parent_start_index(), deleted_region, )) - .await; + .await?; // TODO: this recomputation should not be necessary. self.public_state.worker().reattach_worker_status().await; @@ -3267,7 +3271,7 @@ impl DurableWorkerCtx { // has run through it. self.public_state .worker() - .commit_oplog_or_fenced(CommitLevel::Always) + .commit_oplog_and_update_state(CommitLevel::Always) .await .map_err(WorkerExecutorError::from)?; @@ -3435,7 +3439,7 @@ impl DurableWorkerCtx { // to run, and stopping here avoids opening a database transaction first. self.public_state .worker() - .add_and_commit_oplog_or_fenced(OplogEntry::jump( + .add_and_commit_oplog(OplogEntry::jump( self.entity_parent_start_index(), deleted_region, )) @@ -3452,7 +3456,7 @@ impl DurableWorkerCtx { // refused begin has to stop it; `tx` is dropped unused. self.public_state .worker() - .add_and_commit_oplog_or_fenced(OplogEntry::begin_remote_transaction( + .add_and_commit_oplog(OplogEntry::begin_remote_transaction( tx_id, Some(original_begin_index), )) @@ -3498,7 +3502,7 @@ impl DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; Ok(()) } else { let (_, _) = crate::get_oplog_entry!( @@ -3526,7 +3530,7 @@ impl DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; Ok(()) } else { let (_, _) = crate::get_oplog_entry!( @@ -3655,7 +3659,7 @@ impl DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; self.state.remove_durable_scope(begin_index)?; Ok(()) } @@ -3729,7 +3733,7 @@ impl DurableWorkerCtx { "Manual update failed to lower load-snapshot invocation: {err}" )), ) - .await; + .await?; return Ok(Some(RetryDecision::Immediate)); } }; @@ -3752,7 +3756,7 @@ impl DurableWorkerCtx { "Manual update failed to install invocation context: {err}" )), ) - .await; + .await?; return Ok(Some(RetryDecision::Immediate)); } @@ -3823,7 +3827,7 @@ impl DurableWorkerCtx { .as_context_mut() .data_mut() .on_worker_update_failed(target_revision, Some(error)) - .await; + .await?; Ok(Some(RetryDecision::Immediate)) } else { let component_metadata = @@ -3849,7 +3853,7 @@ impl DurableWorkerCtx { }), ), ) - .await; + .await?; Ok(None) } } @@ -3861,7 +3865,7 @@ impl DurableWorkerCtx { target_revision, Some("Failed to find snapshot data for update".to_string()), ) - .await; + .await?; Ok(Some(RetryDecision::Immediate)) } Err(error) => { @@ -3869,7 +3873,7 @@ impl DurableWorkerCtx { .as_context_mut() .data_mut() .on_worker_update_failed(target_revision, Some(error)) - .await; + .await?; Ok(Some(RetryDecision::Immediate)) } } @@ -4577,7 +4581,7 @@ impl DurableWorkerCtx { target_revision, Some(stringified_error), ) - .await; + .await?; Err(error)? }; @@ -4597,7 +4601,7 @@ impl DurableWorkerCtx { }) }), ) - .await; + .await?; debug!("Finalizing automatic update to revision {target_revision}"); } @@ -4645,7 +4649,7 @@ impl DurableWorkerCtx { self.public_state .worker() .queue_card_revocations_locked(&revoked_card_ids) - .await; + .await?; Ok(()) } @@ -4952,18 +4956,19 @@ impl InvocationHooks for DurableWorkerCtx { }, ) .await - .unwrap_or_else(|err| { - panic!( + .map_err(|err| match err { + OplogError::Fenced(fence) => self.public_state.worker().relinquished_by(fence), + err => panic!( "could not encode agent invocation on {}: {err}", self.agent_id() - ) - }); + ), + })?; self.primary_invocation_start_index = Some(start_index); self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; } Ok(()) } @@ -4984,11 +4989,18 @@ impl InvocationHooks for DurableWorkerCtx { // Deliberately above the dropped-call drain: that drain appends `Cancelled` entries, and // a relinquished agent's oplog belongs to another executor now. Nothing further is - // written for it - not the drain, not an `Error` entry, not a status change. - if matches!(trap_type, TrapType::Interrupt(InterruptKind::ShardLost)) { - self.public_state - .worker() - .mark_relinquished(crate::worker::RelinquishReason::Fenced(None)); + // written for it - not the drain, not an `Error` entry, not a status change - whatever + // the trap was: a revoke latches no fence, so the mark is all that says so, and the shard's + // new owner replays the invocation and records its outcome itself. + let worker = self.public_state.worker(); + let given_up = if matches!(trap_type, TrapType::Interrupt(InterruptKind::ShardLost)) { + worker.relinquish_if_shard_lost(&WorkerExecutorError::Interrupted { + kind: InterruptKind::ShardLost, + }) + } else { + worker.is_relinquished() + }; + if given_up { return RetryDecision::None; } @@ -4996,7 +5008,16 @@ impl InvocationHooks for DurableWorkerCtx { && !self.state.snapshotting_mode && let Err(err) = concurrent::drain_queued_dropped_call_events(self).await { - error!("failed to drain dropped durable calls before invocation failure entry: {err}"); + error!( + error = %err, + "Failed to drain dropped durable calls before the invocation failure entry" + ); + // A `Cancelled` refused by a fence latched during the drain gives the agent up, so + // the stop that follows drops this generation instead of leaving it cached to be + // restarted in place. + self.public_state + .worker() + .relinquish_if_shard_lost(&err.source); return RetryDecision::None; } @@ -5080,10 +5101,16 @@ impl InvocationHooks for DurableWorkerCtx { } Err(error) => panic!("oplog write: {error}"), } - self.public_state + // Refused, like the add above: the agent has been given up. + if self + .public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await + .is_err() + { + return RetryDecision::None; + } true } else { false @@ -5119,8 +5146,17 @@ impl InvocationHooks for DurableWorkerCtx { )), }; - if let Some(entry) = oplog_entry { - self.public_state.worker().add_and_commit_oplog(entry).await; + // Refused, the agent has been given up: no failure is published for its invocation, which + // the shard's new owner runs. + if let Some(entry) = oplog_entry + && self + .public_state + .worker() + .add_and_commit_oplog(entry) + .await + .is_err() + { + return RetryDecision::None; }; let latest_status = self @@ -5289,14 +5325,17 @@ impl InvocationHooks for DurableWorkerCtx { component_revision, ) .await - .unwrap_or_else(|err| { - panic!("could not encode function result for {full_function_name}: {err}") - }); + .map_err(|err| match err { + OplogError::Fenced(fence) => self.public_state.worker().relinquished_by(fence), + err => { + panic!("could not encode function result for {full_function_name}: {err}") + } + })?; self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; // Bump the read-only cache epoch after the // `AgentInvocationFinished` entry is committed, but *before* @@ -5437,15 +5476,22 @@ impl UpdateManagement for DurableWorkerCtx { &self, target_revision: ComponentRevision, details: Option, - ) { + ) -> Result<(), WorkerExecutorError> { + // A given-up agent's update is settled by the shard's new owner. A revoke latches no + // fence, so the storage would still accept this entry, and it would drop the update there. + let worker = self.public_state.worker(); + if worker.is_relinquished() { + return Err(worker.relinquish_error()); + } let entry = OplogEntry::failed_update(target_revision, details.clone()); - self.public_state.worker().add_and_commit_oplog(entry).await; + worker.add_and_commit_oplog(entry).await?; warn!( "Worker failed to update to {}: {}, update attempt aborted", target_revision, details.unwrap_or_else(|| "?".to_string()) ); + Ok(()) } async fn on_worker_update_succeeded( @@ -5455,9 +5501,13 @@ impl UpdateManagement for DurableWorkerCtx { new_active_plugins: HashSet< golem_common::base_model::environment_plugin_grant::EnvironmentPluginGrantId, >, - ) { + ) -> Result<(), WorkerExecutorError> { info!("Worker update to {} finished successfully", target_revision); let worker = self.public_state.worker(); + // As for a failed update: the outcome is the shard's new owner's to record. + if worker.is_relinquished() { + return Err(worker.relinquish_error()); + } worker .persist_successful_update( &self.linear_memory, @@ -5465,7 +5515,8 @@ impl UpdateManagement for DurableWorkerCtx { new_component_size, new_active_plugins, ) - .await; + .await?; + Ok(()) } } @@ -6041,7 +6092,14 @@ impl ExternalOperations for DurableWorkerCtx { error, stderr: store.as_context().data().get_public_state().event_service().get_last_invocation_errors(), }), - TrapType::Interrupt(kind) => Self::fixed_decision_for_trap_type(&TrapType::Interrupt(kind)), + TrapType::Interrupt(kind) => { + // `on_invocation_failure` is skipped on this path, + // so a lost shard is given up here: its `None` + // decision alone would stop the agent as an + // ordinary one, left cached to restart in place. + worker.relinquish_if_shard_lost(&WorkerExecutorError::Interrupted { kind }); + Self::fixed_decision_for_trap_type(&TrapType::Interrupt(kind)) + } TrapType::Exit => break Err(WorkerExecutorError::runtime("Process exited during snapshot replay")), } } @@ -6055,9 +6113,12 @@ impl ExternalOperations for DurableWorkerCtx { if decision == RetryDecision::None { // Like the invocation loop, permanently fail the // durable Stream Session of an invocation that was - // interrupted by a crash and cannot be retried. + // interrupted by a crash and cannot be retried. Not + // for an agent given up here: the shard's new owner + // resumes that invocation. if uses_streams && store.as_context().data().durable_ctx().is_live() + && !worker.is_relinquished() { let _ = worker .fail_durable_streaming_session( @@ -6258,7 +6319,7 @@ impl ExternalOperations for DurableWorkerCtx { "Automatic update failed: {error}" )), ) - .await; + .await?; debug!( "Retrying prepare_instance after failed update attempt" @@ -6819,16 +6880,18 @@ fn recovered_status( /// /// `ShardingNotReady` means the agent's shard left this executor's assignment while recovery was /// running: a later delivery revoked it, and the worker was refused an epoch rather than opened -/// unfenced. The agent belongs to the shard's new owner, which recovers it there. This is a skip, -/// like an oplog that is gone. Failing the whole assignment for it would stop every other agent in -/// the scan from being resumed. Any other restart failure still fails the assignment. +/// unfenced. `OplogFenced` means another executor claimed the agent's oplog at a newer epoch +/// before this one wrote to it. Either way the agent belongs to the shard's new owner, which +/// recovers it there. This is a skip, like an oplog that is gone. Failing the whole assignment for +/// it would stop every other agent in the scan from being resumed. Any other restart failure still +/// fails the assignment. fn recovered_restart( owned_agent_id: &OwnedAgentId, restarted: Result, ) -> Result<(), anyhow::Error> { match restarted { Ok(_) => Ok(()), - Err(WorkerExecutorError::ShardingNotReady) => { + Err(WorkerExecutorError::ShardingNotReady | WorkerExecutorError::OplogFenced { .. }) => { debug!(agent_id = %owned_agent_id, "Worker's shard left the assignment during shard-assignment recovery; skipping agent"); Ok(()) } @@ -8800,6 +8863,18 @@ mod tests { ) .is_ok() ); + let agent = recovered_agent(); + assert!( + recovered_restart::<()>( + &agent, + Err(WorkerExecutorError::oplog_fenced( + agent.agent_id.clone(), + 2, + Some(3) + )) + ) + .is_ok() + ); let error = recovered_restart::<()>( &recovered_agent(), diff --git a/golem-worker-executor/src/durable_host/permissions/mod.rs b/golem-worker-executor/src/durable_host/permissions/mod.rs index b82c4049ee..4d8e6d5648 100644 --- a/golem-worker-executor/src/durable_host/permissions/mod.rs +++ b/golem-worker-executor/src/durable_host/permissions/mod.rs @@ -986,7 +986,7 @@ where card: created.clone(), wallet_generation, }) - .await; + .await?; } else { return Err(anyhow!( "replayed runtime permission-card creation {card_id} is missing its CardDerived audit event" @@ -1702,7 +1702,7 @@ async fn complete_source_card_transfer( installed_card.card_id(), target_holder, )) - .await; + .await?; Ok(()) } @@ -1743,7 +1743,7 @@ async fn ensure_source_card_transfer_started( target_holder.clone(), Some(ctx.state.wallet_generation), )) - .await; + .await?; Ok(()) } @@ -1795,7 +1795,7 @@ async fn execute_source_card_transfer( card: installed_card.clone(), wallet_generation: Some(ctx.state.wallet_generation), }) - .await; + .await?; } ctx.public_state @@ -1809,7 +1809,7 @@ async fn execute_source_card_transfer( transfer.target_holder(), ), )) - .await; + .await?; } complete_source_card_transfer( @@ -2133,7 +2133,7 @@ pub(super) async fn complete_pending_source_card_transfers( agent_id: retry.target_agent_id, }), )) - .await; + .await?; } Ok(()) @@ -2794,7 +2794,7 @@ impl permissions_wallet::Host for DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; } let start_index = handle.start_index(); diff --git a/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs index 4597f04f07..69121f3ada 100644 --- a/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs @@ -1450,7 +1450,7 @@ impl HostWasmRpc for DurableWorkerCtx { self.public_state .worker() .commit_oplog_and_update_state(CommitLevel::DurableOnly) - .await; + .await?; } let auth_ctx = handle.take_agent_auth_ctx(); @@ -2736,7 +2736,7 @@ async fn run_invoke_and_await( ctx.public_state .worker() .commit_oplog_and_update_state(CommitLevel::DurableOnly) - .await; + .await?; } let either_result = futures::future::select( @@ -2864,7 +2864,7 @@ async fn run_invoke( ctx.public_state .worker() .commit_oplog_and_update_state(CommitLevel::DurableOnly) - .await; + .await?; } let result = ctx diff --git a/golem-worker-executor/src/services/active_agents/mod.rs b/golem-worker-executor/src/services/active_agents/mod.rs index 52f37fcbe5..7da813252d 100644 --- a/golem-worker-executor/src/services/active_agents/mod.rs +++ b/golem-worker-executor/src/services/active_agents/mod.rs @@ -974,7 +974,8 @@ impl ActiveAgents { continue; }; - worker.queue_card_revocations(&affected_card_ids).await; + // A refusal has already given the agent up; the other agents are still notified. + let _ = worker.queue_card_revocations(&affected_card_ids).await; } } diff --git a/golem-worker-executor/src/services/oplog/compressed.rs b/golem-worker-executor/src/services/oplog/compressed.rs index f1465a58af..d29b012290 100644 --- a/golem-worker-executor/src/services/oplog/compressed.rs +++ b/golem-worker-executor/src/services/oplog/compressed.rs @@ -78,6 +78,92 @@ where } } +/// Appends one already-serialized compressed chunk, retrying transient failures like +/// [`retry_storage_op`]. A permanent failure is reconciled against storage before it is treated +/// as fatal, because this append is not protected by a shard epoch (it is driven by whichever +/// executor's transfer fiber is running, primary-owner or not) and its background task can be +/// aborted between steps - including after this append lands but before the `drop_source_prefix` +/// that would have advanced the source past it. The owner's next transfer then chunks from the +/// same unadvanced point, so a chunk it writes under an id that already exists holds the identical +/// entries and bytes. A duplicate-id failure whose stored content matches what this attempt would +/// have written is that resumed transfer catching up, not corruption, and is treated as success +/// rather than panicking on the storage's key conflict. +async fn append_compressed_chunk( + retry_config: &RetryConfig, + indexed_storage: &(dyn IndexedStorage + Send + Sync), + namespace: &IndexedStorageNamespace, + key: &str, + id: u64, + value: Vec, +) { + let mut attempts = 0u32; + loop { + attempts += 1; + let error = match indexed_storage + .with_entity("compressed_oplog", "append", "compressed_entry") + .append_raw(namespace.clone(), key, id, value.clone(), None) + .await + { + Ok(()) => return, + Err(error) => error, + }; + + if let IndexedStorageError::Transient(msg) = &error { + if let Some(delay) = get_delay(retry_config, attempts) { + record_oplog_storage_retry("compressed_append"); + warn!( + op = "compressed_append", + key = key, + attempt = attempts, + delay_ms = delay.as_millis() as u64, + "Transient indexed storage error, retrying: {msg}" + ); + tokio::time::sleep(delay).await; + continue; + } + panic!( + "Indexed storage operation 'compressed_append' failed for key '{key}' after {attempts} attempts: Transient storage error: {msg}" + ); + } + + if stored_chunk_matches(retry_config, indexed_storage, namespace, key, id, &value).await { + return; + } + panic!("Indexed storage operation 'compressed_append' failed for key '{key}': {error}"); + } +} + +/// Reads back the chunk stored at `id` and compares it byte-for-byte with `expected`. Used only to +/// tell a resumed transfer's harmless repeat write apart from a genuine conflict - see +/// [`append_compressed_chunk`]. +async fn stored_chunk_matches( + retry_config: &RetryConfig, + indexed_storage: &(dyn IndexedStorage + Send + Sync), + namespace: &IndexedStorageNamespace, + key: &str, + id: u64, + expected: &[u8], +) -> bool { + let actual = retry_storage_op(retry_config, "compressed_append_reconcile", key, || { + let namespace = namespace.clone(); + async move { + indexed_storage + .with_entity( + "compressed_oplog", + "compressed_append_reconcile", + "compressed_entry", + ) + .read_raw(namespace, key, id, id) + .await + } + }) + .await; + actual + .into_iter() + .find(|(actual_id, _)| *actual_id == id) + .is_some_and(|(_, bytes)| bytes == expected) +} + #[derive(Debug)] pub struct CompressedOplogArchiveService { indexed_storage: Arc, @@ -487,27 +573,22 @@ impl OplogArchive for CompressedOplogArchive { total_bytes += compressed_chunk.compressed_data.len() as u64; { - let is = self.indexed_storage.clone(); - let agent_id_clone = self.agent_id.clone(); - let agent_mode = self.agent_mode; - let level = self.level; - let key = self.key.clone(); + let ns = IndexedStorageNamespace::CompressedOpLog { + agent_id: self.agent_id.clone(), + agent_mode: self.agent_mode, + level: self.level, + }; let last_id_val: u64 = last_id.into(); - retry_storage_op(&self.retry_config, "compressed_append", &key, || { - let is = is.clone(); - let ns = IndexedStorageNamespace::CompressedOpLog { - agent_id: agent_id_clone.clone(), - agent_mode, - level, - }; - let key = key.clone(); - let chunk = compressed_chunk.clone(); - async move { - is.with_entity("compressed_oplog", "append", "compressed_entry") - .append(ns, &key, last_id_val, &chunk, None) - .await - } - }) + let value = serialize(&compressed_chunk) + .unwrap_or_else(|err| panic!("failed to serialize oplog chunk: {err}")); + append_compressed_chunk( + &self.retry_config, + self.indexed_storage.as_ref(), + &ns, + &self.key, + last_id_val, + value, + ) .await; } } diff --git a/golem-worker-executor/src/services/oplog/mod.rs b/golem-worker-executor/src/services/oplog/mod.rs index f75a3babff..ac792de14e 100644 --- a/golem-worker-executor/src/services/oplog/mod.rs +++ b/golem-worker-executor/src/services/oplog/mod.rs @@ -49,7 +49,6 @@ use std::collections::BTreeMap; use std::fmt::{Debug, Display, Formatter}; use std::marker::PhantomData; use std::ops::Deref; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -1149,9 +1148,11 @@ impl OplogServiceOps for O {} #[derive(Clone)] struct OpenOplogEntry { pub oplog: Weak, - pub initial: Arc, /// Identifies this insertion, so that the remover the oplog runs when it is dropped removes - /// this entry and not a replacement cached under the same agent after it. + /// this entry and not a replacement cached under the same agent after it. Also identifies, + /// to the call whose closure built this entry, that it is the one that built it: comparing + /// against the opener's own token (not a flag shared by every concurrent reader of the cache + /// entry) is race-free, since each opener allocates a distinct token before racing to insert. pub token: Arc<()>, /// The epoch the opener that constructed this handle asked it to assert. pub requested_epoch: Option, @@ -1161,7 +1162,6 @@ impl OpenOplogEntry { pub fn new(oplog: Arc, token: Arc<()>, requested_epoch: Option) -> Self { Self { oplog: Arc::downgrade(&oplog), - initial: Arc::new(AtomicBool::new(true)), token, requested_epoch, } @@ -1226,7 +1226,15 @@ impl OpenOplogs { .await .unwrap(); if let Some(oplog) = entry.oplog.upgrade() { - let just_constructed = entry.initial.swap(false, Ordering::AcqRel); + // Whether *this* call's closure is the one that built the cached entry, not + // whether it merely observed it first: every concurrent opener racing on the + // same key gets a clone of the same entry back, so a shared flag here would + // let a newer-epoch opener win a race against the actual constructor and skip + // the older-generation eviction below, handing it a stale handle without ever + // recording its own epoch. `token` is a fresh allocation per opener, and only + // the constructing closure's copy ends up stored on the entry, so identity by + // pointer is decided at construction time, not by scheduling order. + let just_constructed = Arc::ptr_eq(&entry.token, &token); let oplog = if just_constructed { unsafe { let ptr = Arc::into_raw(oplog); @@ -1271,7 +1279,17 @@ impl OpenOplogs { break oplog; } else { - self.oplogs.remove(agent_id).await; + // Scoped to this entry's own token, like the eviction above: a concurrent + // opener can already have replaced this dead weak reference with a Pending + // construction of its own by the time we get here, and `remove_if_cached` + // never touches a Pending entry. An unconditional remove-by-key would delete + // that in-flight Pending marker instead, leaving the key looking empty to a + // third opener, which would then start a second, independent construction - + // two live oplog actors writing the same initial index at the same epoch, one + // of them aborting on the storage's unique-key conflict. + self.oplogs + .remove_if_cached(agent_id, |cached| Arc::ptr_eq(&cached.token, &entry.token)) + .await; continue; } } diff --git a/golem-worker-executor/src/services/oplog/multilayer.rs b/golem-worker-executor/src/services/oplog/multilayer.rs index a949ed0a2d..d3a43d8a6f 100644 --- a/golem-worker-executor/src/services/oplog/multilayer.rs +++ b/golem-worker-executor/src/services/oplog/multilayer.rs @@ -1318,6 +1318,22 @@ trait BackgroundTransfer { async fn verify_target(&self, entries: &[(OplogIndex, OplogEntry)]); async fn drop_source_prefix(&self, last_dropped_id: OplogIndex); + /// `try_abort_transfer` can land between any two `.await`s here, including between + /// `append_target` and `drop_source_prefix`: a `JoinHandle::abort` takes effect at whichever + /// suspension point the task is next parked at, not at a step boundary this trait controls. + /// So a target chunk can already be durable while the source that fed it has not yet been + /// trimmed. That is only safe because the target append is written to be replayed: the next + /// transfer starts from the same untrimmed source position and chunks from there, so every + /// chunk it shares an id with has the identical bytes, and the archive backing `append_target` + /// (the compressed layer; a blob-backed one overwrites by path and is idempotent by + /// construction) reconciles a duplicate-id write against what is already stored instead of + /// treating it as a conflict. A next transfer that covers more entries ends the aborted run's + /// trailing partial chunk at a later id instead: the two overlap with identical entries, which + /// reads tolerate, and the earlier one goes with the layer's next `drop_prefix` past it. Sequencing the steps + /// behind a cooperative, checked-between-steps cancellation instead of a hard abort would + /// also close this window, but the archive already has to tolerate a replayed append for + /// other reasons (retried indeterminate writes), so leaning on that here avoids a second + /// cancellation mechanism. async fn run(&self) { let entries = self.read_source().await; match entries.last() { diff --git a/golem-worker-executor/src/services/oplog/plugin.rs b/golem-worker-executor/src/services/oplog/plugin.rs index ff4816ad5a..d2cfaebc56 100644 --- a/golem-worker-executor/src/services/oplog/plugin.rs +++ b/golem-worker-executor/src/services/oplog/plugin.rs @@ -18,7 +18,7 @@ use crate::services::component::ComponentService; use crate::services::oplog::{ CommitLevel, DurableStreamBatchBuilder, IndexedReservedStartBuilder, OpenOplogs, Oplog, OplogAddReceipt, OplogConstructor, OplogError, OplogFence, OplogService, OrderedOplogStart, - ReservedRawStartBuilder, + ReservedRawStartBuilder, downcast_oplog, }; use crate::services::shard::ShardService; use crate::services::worker_activator::WorkerActivator; @@ -877,8 +877,12 @@ impl OplogService for ForwardingOplogService { pub struct ForwardingOplog { inner: Arc, jobs: tokio::sync::mpsc::UnboundedSender, - actor: JoinHandle<()>, - timer: Option>, + /// `Mutex`-guarded (not owned outright) so [`try_join_background_work`] can take both + /// handles out through a shared reference and await them, the same way `Drop` aborts them + /// through a shared method (`JoinHandle::abort` only needs `&self`). Never held across an + /// `.await`. + actor: std::sync::Mutex>>, + timer: std::sync::Mutex>>, close_fn: Option>, } @@ -917,6 +921,10 @@ enum ForwardingJob { }, /// Periodic tick from the timer task: runs locality recovery and a time-based flush. Tick, + /// Sent only by [`try_join_background_work`]: drains no further jobs after this one, so the + /// actor exits once every job already queued ahead of it - including a stray `Tick` the timer + /// enqueued in the instant before it was stopped - has been processed. + Shutdown, #[cfg(test)] Inspect { done: tokio::sync::oneshot::Sender, @@ -1149,6 +1157,7 @@ impl ForwardingOplog { )) .await; } + ForwardingJob::Shutdown => break, #[cfg(test)] ForwardingJob::Inspect { done } => { let _ = done.send(ForwardingStateSnapshot { @@ -1176,8 +1185,8 @@ impl ForwardingOplog { Self { inner, jobs, - actor, - timer: Some(timer), + actor: std::sync::Mutex::new(Some(actor)), + timer: std::sync::Mutex::new(Some(timer)), close_fn: Some(close_fn), } } @@ -1208,8 +1217,9 @@ impl ForwardingOplog { /// Enqueues a job for the actor task and awaits its reply. /// - /// Panics if the actor task is gone: the actor is only aborted from `Drop` (when no caller - /// can be in flight anymore), so a missing reply means the actor itself panicked and the + /// Panics if the actor task is gone: the actor is aborted only from `Drop`, and stopped + /// cooperatively only by `try_join_background_work`, both of which run only once no caller + /// can still be in flight - so a missing reply means the actor itself panicked and the /// oplog's state is no longer trustworthy. async fn run_job( &self, @@ -1237,15 +1247,50 @@ impl Drop for ForwardingOplog { if let Some(close_fn) = self.close_fn.take() { close_fn(); } - if let Some(timer) = self.timer.take() { + if let Some(timer) = self.timer.get_mut().unwrap().take() { timer.abort(); } // In-flight `Oplog` calls borrow `self`, so at this point no caller can be awaiting a // job reply anymore and aborting the actor cannot lose an observed operation. Dropping // the actor's state aborts all background monitor tasks (they are // `AbortOnDropJoinHandle`s), preventing them from outliving this oplog and causing - // resource contention. - self.actor.abort(); + // resource contention. A handle that already went through `try_join_background_work` has + // taken both out already, so there is nothing left here to abort. + if let Some(actor) = self.actor.get_mut().unwrap().take() { + actor.abort(); + } + } +} + +/// Stops this handle's periodic commit timer and forwarding actor and waits for both to actually +/// finish, so no job either one is holding - or that the timer enqueues in the instant before it +/// stops - can still be running once this returns. Does nothing for an oplog with no forwarding +/// layer. +/// +/// Unlike `Drop` (which only requests cancellation: by the time it runs no caller can still be +/// waiting on a job reply, and a live oplog's own epoch fences anything the actor is still +/// writing), a handle built for a fork target asserts no epoch at all - it is closed and hands off +/// to the target's real owner before that owner opens its own primary oplog at its own epoch. A +/// periodic checkpoint commit the actor is still running when the caller moves on would land, +/// unfenced, into storage the owner may already be writing into. Aborting the timer stops it from +/// scheduling further ticks; the actor is never aborted, only sent a `Shutdown` job and awaited, +/// so it drains everything already queued ahead of that job - including a tick the timer sent in +/// the instant before it was stopped - before exiting. +pub(crate) async fn try_join_background_work(this: &Arc) { + let Some(this) = downcast_oplog::(this) else { + return; + }; + let timer = this.timer.lock().unwrap().take(); + if let Some(timer) = timer { + timer.abort(); + let _ = timer.await; + } + let actor = this.actor.lock().unwrap().take(); + if let Some(actor) = actor { + // Ignored: a send failure means the actor is already gone (panicked), in which case + // there is nothing left to drain and awaiting its handle below still completes. + let _ = this.jobs.send(ForwardingJob::Shutdown); + let _ = actor.await; } } diff --git a/golem-worker-executor/src/services/oplog/primary.rs b/golem-worker-executor/src/services/oplog/primary.rs index 1de0b391b8..7b11647b86 100644 --- a/golem-worker-executor/src/services/oplog/primary.rs +++ b/golem-worker-executor/src/services/oplog/primary.rs @@ -263,9 +263,25 @@ async fn retry_oplog_append( .await { Some(true) => return Ok(()), - Some(false) => panic!( - "Indexed storage operation '{op_name}' failed for key '{key}' and the indeterminate write did not match storage: {error}" - ), + Some(false) => { + // The stored content differs from what this attempt sent - the only + // legitimate way that happens is a new owner having already written those + // same indices. Repeat the write once as a probe: the backends check the + // epoch inside the same transaction as the insert, so a shard that has + // moved on is fenced before the insert is even attempted. A same-epoch + // conflict instead fails the probe's insert (still fatal, below) - the + // mismatch is unexplained and not safe to paper over. + if let Some(epoch) = shard_epoch + && let Err(fenced @ IndexedStorageError::Fenced { .. }) = append + .write(indexed_storage, namespace, api_name, key, Some(epoch)) + .await + { + return Err(fenced); + } + panic!( + "Indexed storage operation '{op_name}' failed for key '{key}' and the indeterminate write did not match storage: {error}" + ) + } None => {} } } @@ -1351,6 +1367,10 @@ impl PrimaryOplog { match job { OplogJob::Add { entry, done } => { record_oplog_call("add"); + if let Err(error) = state.refuse_if_fenced() { + let _ = done.send(Err(error)); + continue; + } let idx = state.push(entry); // A threshold commit failing must fail the `add` that triggered it: the // caller would otherwise be told its entry landed when the batch it was @@ -1363,6 +1383,10 @@ impl PrimaryOplog { } OplogJob::AddDurableStreamBatch { make_batch, done } => { record_oplog_call("add_durable_stream_batch"); + if let Err(error) = state.refuse_if_fenced() { + let _ = done.send(Err(error)); + continue; + } let first_index = state.last_oplog_idx.next(); let records = make_batch(first_index); let serialized = records @@ -1400,6 +1424,10 @@ impl PrimaryOplog { done, } => { record_oplog_call("add_pair"); + if let Err(error) = state.refuse_if_fenced() { + let _ = done.send(Err(error)); + continue; + } let first_idx = state.push(start); let second = make_second(first_idx); let second_idx = state.push(second); @@ -1428,6 +1456,13 @@ impl PrimaryOplog { // `guard`: this actor future must stay `Send` for `tokio::spawn`, so a // refactor holding the guard across an `.await` is rejected rather than // silently breaking ordering. Do not move `drop(guard)` before `push`. + // + // A fenced oplog refuses before reserving, so no upload is started for a + // `Start` that can never be written. + if let Err(error) = state.refuse_if_fenced() { + let _ = done.send(Err(error)); + continue; + } let result = { let ReservedPayload { raw, @@ -1459,6 +1494,10 @@ impl PrimaryOplog { done, } => { record_oplog_call("add_start_with_indexed_reserved_raw_payload"); + if let Err(error) = state.refuse_if_fenced() { + let _ = done.send(Err(error)); + continue; + } let result = build_request(state.last_oplog_idx.next()).and_then( |(serialized_request, build_start)| { let ReservedPayload { @@ -1890,10 +1929,12 @@ impl PrimaryOplogState { ) -> Result, OplogError> { record_oplog_call("append"); - // Already refused once: fail fast rather than re-asking the storage for every entry the - // guest goes on to produce before it notices it has been given up. + // Already refused once: fail fast rather than re-asking the storage. Only entries buffered + // before the fence latched can reach here, and they go back where they were. if let Some(fence) = self.fence.get() { - return Err(OplogError::Fenced(fence.clone())); + let fence = fence.clone(); + self.retain_refused(entries); + return Err(OplogError::Fenced(fence)); } // Commit barrier: every deferred external payload reserved during this session must be @@ -1938,7 +1979,7 @@ impl PrimaryOplogState { agent_id: self.owned_agent_id.agent_id(), agent_mode: self.agent_mode, }; - retry_oplog_append( + let appended = retry_oplog_append( &self.retry_config, self.indexed_storage.as_ref(), &namespace, @@ -1949,15 +1990,14 @@ impl PrimaryOplogState { self.shard_epoch, ) .await - .map_err(|err| Self::as_oplog_error(&self.owned_agent_id, err)) - .inspect_err(|err| { - if let OplogError::Fenced(fence) = err { - // The drained entries are dropped: this oplog is not ours to write. Nothing else - // is undone - the commit barrier above already awaited every payload the batch - // referenced, so those blobs are durable and stay behind with no entry pointing at - // them. The epoch the storage holds is reported, so a shard manager whose state - // lost history can mint above it. Warned only when it latches: each write after - // that fails fast on the latch without reaching the storage. + .map_err(|err| Self::as_oplog_error(&self.owned_agent_id, err)); + if let Err(error) = appended { + if let OplogError::Fenced(fence) = &error { + // The commit barrier above already awaited every payload the batch referenced, so + // those blobs are durable and stay behind with no stored entry pointing at them. + // The epoch the storage holds is reported, so a shard manager whose state lost + // history can mint above it. Warned only when it latches: each write after that + // fails fast on the latch without reaching the storage. if self.fence.set(fence.clone()).is_ok() { warn!( agent_id = %self.owned_agent_id, @@ -1969,8 +2009,10 @@ impl PrimaryOplogState { if let Some(observer) = &self.fence_observer { observer.fenced(fence); } + self.retain_refused(pairs.into_iter().map(|(_, entry)| entry)); } - })?; + return Err(error); + } record_storage_bytes_written( STORAGE_TYPE_OPLOG, @@ -1993,6 +2035,31 @@ impl PrimaryOplogState { )) } + /// Refuses a new write once the fence has latched, before anything is buffered or reserved. + /// + /// Without it an add below the commit threshold would only buffer, answer with an index, and + /// report a write that can never reach the storage. + fn refuse_if_fenced(&self) -> Result<(), OplogError> { + match self.fence.get() { + Some(fence) => Err(OplogError::Fenced(fence.clone())), + None => Ok(()), + } + } + + /// Puts entries a fenced append turned away back at the head of the buffer, where `commit` + /// drained them from. + /// + /// Every index this oplog has handed out stays readable from it: `last_oplog_idx` is not + /// rolled back, and the reader maps the buffer from `last_committed_idx`. A reader that took + /// `current_oplog_index` before the refusal and reads after it would otherwise find a gap and + /// fail-stop the executor. The entries are never sent again, because every later append fails + /// on the latch, and the buffer cannot grow, because every later add is refused. + fn retain_refused(&mut self, entries: impl IntoIterator) { + let mut restored: VecDeque = entries.into_iter().collect(); + restored.append(&mut self.buffer); + self.buffer = restored; + } + /// Commits if the buffer is over the threshold. Separated out so the actor arms can fold a /// threshold-commit failure into the job that triggered it. async fn maybe_commit(&mut self) -> Result<(), OplogError> { diff --git a/golem-worker-executor/src/services/oplog/tests.rs b/golem-worker-executor/src/services/oplog/tests.rs index 532b52516d..c0508e9585 100644 --- a/golem-worker-executor/src/services/oplog/tests.rs +++ b/golem-worker-executor/src/services/oplog/tests.rs @@ -489,10 +489,18 @@ enum InjectedAppendFailure { CommitThenIndeterminate, CommitDifferentThenIndeterminate, CommitPrefixThenIndeterminate, + /// Refuses the write as a stale epoch would, naming the epoch one above whatever was + /// asserted. Used to simulate the storage fencing the reconciliation probe + /// `retry_oplog_append` repeats after a genuine indeterminate-write mismatch (F16). + Fenced, } impl InjectedAppendFailure { - fn before_write_error(self) -> Option { + fn before_write_error( + self, + key: &str, + shard_epoch: Option, + ) -> Option { match self { Self::IndeterminateBeforeWrite => Some(IndexedStorageError::Indeterminate( "injected connection loss".to_string(), @@ -503,6 +511,11 @@ impl InjectedAppendFailure { Self::PermanentBeforeWrite => Some(IndexedStorageError::Other( "injected permanent failure".to_string(), )), + Self::Fenced => Some(IndexedStorageError::Fenced { + key: key.to_string(), + expected: shard_epoch.unwrap_or_default(), + actual: shard_epoch.map(|epoch| ShardEpoch(epoch.0 + 1)), + }), _ => None, } } @@ -517,7 +530,8 @@ impl InjectedAppendFailure { )), Self::IndeterminateBeforeWrite | Self::TransientBeforeWrite - | Self::PermanentBeforeWrite => unreachable!(), + | Self::PermanentBeforeWrite + | Self::Fenced => unreachable!(), } } } @@ -719,7 +733,7 @@ impl IndexedStorage for ReadCountingIndexedStorage { .unwrap() .pop_front() .unwrap_or(InjectedAppendFailure::None); - if let Some(error) = failure.before_write_error() { + if let Some(error) = failure.before_write_error(key, shard_epoch) { return Err(error); } if matches!( @@ -775,7 +789,7 @@ impl IndexedStorage for ReadCountingIndexedStorage { .unwrap() .pop_front() .unwrap_or(InjectedAppendFailure::None); - if let Some(error) = failure.before_write_error() { + if let Some(error) = failure.before_write_error(key, shard_epoch) { return Err(error); } let pairs = if matches!( @@ -1641,6 +1655,14 @@ async fn append_reconciliation_service( async fn create_append_reconciliation_oplog( service: &PrimaryOplogService, name: &str, +) -> Arc { + create_append_reconciliation_oplog_with_epoch(service, name, None).await +} + +async fn create_append_reconciliation_oplog_with_epoch( + service: &PrimaryOplogService, + name: &str, + shard_epoch: Option, ) -> Arc { let account_id = AccountId::new(); let environment_id = EnvironmentId::new(); @@ -1663,7 +1685,7 @@ async fn create_append_reconciliation_oplog( make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), - None, + shard_epoch, ) .await } @@ -2101,6 +2123,39 @@ async fn differing_read_back_after_indeterminate_append_remains_fatal(_tracing: assert_eq!(indexed_storage.reads(), 1); } +/// Same mismatch as `differing_read_back_after_indeterminate_append_remains_fatal`, except this +/// writer asserts a shard epoch and the mismatch is explained: a new owner already wrote those +/// indices. The reconciliation probe this fences (F16) must return `Fenced` and let the caller +/// give up the agent, rather than panicking and aborting the whole - otherwise still live - +/// executor process (`panic = "abort"`). +#[test] +async fn differing_read_back_on_a_moved_shard_is_fenced_instead_of_panicking(_tracing: &Tracing) { + let indexed_storage = Arc::new(ReadCountingIndexedStorage::new()); + let service = append_reconciliation_service(indexed_storage.clone()).await; + let oplog = create_append_reconciliation_oplog_with_epoch( + &service, + "different-append-read-back-fenced", + Some(ShardEpoch(5)), + ) + .await; + indexed_storage.reset(); + indexed_storage.reset_append_observations(); + indexed_storage.inject_append_many_failures([ + InjectedAppendFailure::CommitDifferentThenIndeterminate, + InjectedAppendFailure::Fenced, + ]); + + oplog.add(OplogEntry::suspend()).await.expect("oplog write"); + let result = oplog.commit(CommitLevel::Always).await; + + assert!( + matches!(result, Err(OplogError::Fenced(_))), + "expected the reconciliation probe to surface a fence instead of panicking, got {result:?}" + ); + // The original attempt, then the reconciliation probe once the read-back mismatched. + assert_eq!(indexed_storage.append_many_attempts(), 2); +} + #[test] async fn open_add_and_read_back(_tracing: &Tracing) { let indexed_storage = Arc::new(InMemoryIndexedStorage::new()); @@ -7705,12 +7760,11 @@ async fn a_fenced_oplog_is_not_handed_out_again_while_it_is_still_held(_tracing: ); } -/// What every write that gates a side effect relies on: an add's answer is no evidence the entry -/// was written, only the latch read after the commit is. A below-threshold add on a moved shard -/// buffers and succeeds, before and after the fence latches, and the refused commit is what -/// latches it. +/// A below-threshold add on a moved shard only buffers, so nothing refuses it until the commit; +/// the refused commit latches the fence, and from then on the add itself is refused rather than +/// buffered under an index that could never reach the storage. #[test] -async fn a_below_threshold_add_on_a_moved_shard_is_latched_by_the_refused_commit( +async fn a_below_threshold_add_on_a_moved_shard_is_refused_once_the_commit_latches_the_fence( _tracing: &Tracing, ) { let tempdir = tempfile::TempDir::new().unwrap(); @@ -7762,10 +7816,13 @@ async fn a_below_threshold_add_on_a_moved_shard_is_latched_by_the_refused_commit assert_eq!(fence.expected_epoch, ShardEpoch(8)); assert_eq!(fence.actual_epoch, Some(ShardEpoch(9))); - stale - .add(OplogEntry::exited().rounded()) - .await - .expect("a latched fence does not stop a below-threshold add"); + assert!( + matches!( + stale.add(OplogEntry::exited().rounded()).await, + Err(OplogError::Fenced(_)) + ), + "a latched fence refuses a below-threshold add" + ); assert!(matches!( stale.commit(CommitLevel::Always).await, Err(OplogError::Fenced(_)) @@ -7980,6 +8037,96 @@ async fn wait_for_replicas_does_not_report_a_fenced_flush_as_durable(_tracing: & ); } +#[test] +async fn a_fenced_oplog_refuses_new_adds_and_keeps_the_indices_it_handed_out_readable( + _tracing: &Tracing, +) { + let tempdir = tempfile::TempDir::new().unwrap(); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: ComponentId::new(), + agent_id: "half-alive".into(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let losing_executor = fencing_oplog_service(&tempdir, "half-alive").await; + let owning_executor = fencing_oplog_service(&tempdir, "half-alive").await; + + let loser = losing_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(8)), + ) + .await; + let owner = owning_executor + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id.clone(), account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + Some(golem_common::model::ShardEpoch(9)), + ) + .await; + owner.add(OplogEntry::suspend().rounded()).await.unwrap(); + owner.commit(CommitLevel::Always).await.unwrap(); + + // Below the commit threshold both adds only buffer, and each is answered with an index. + let first = loser.add(OplogEntry::suspend().rounded()).await.unwrap(); + loser.add(OplogEntry::exited().rounded()).await.unwrap(); + // A reader takes the horizon before the storage refuses the batch and reads after it, as a + // durable session or a fork running beside the invocation loop does. + let horizon = loser.current_oplog_index().await; + assert!(matches!( + loser.commit(CommitLevel::Always).await, + Err(OplogError::Fenced(_)) + )); + let entries = loser + .read_exact(first, horizon.as_u64() - first.as_u64() + 1) + .await; + assert_eq!( + entries.len(), + 2, + "an index handed out before the refusal must still be readable after it" + ); + + // Once latched, an add below the threshold is refused instead of buffered under an index + // that could never reach the storage. + assert!(matches!( + loser.add(OplogEntry::suspend().rounded()).await, + Err(OplogError::Fenced(_)) + )); + assert!(matches!( + loser + .add_pair( + OplogEntry::suspend().rounded(), + Box::new(|_| OplogEntry::exited().rounded()) + ) + .await, + Err(OplogError::Fenced(_)) + )); + assert_eq!( + loser.current_oplog_index().await, + horizon, + "a refused add must not take an index" + ); + assert!(matches!( + loser.commit(CommitLevel::Always).await, + Err(OplogError::Fenced(_)) + )); + assert_eq!( + owner.length().await, + 1, + "the losing executor must not have appended to the owner's oplog" + ); +} + #[test] async fn an_opener_at_a_newer_epoch_is_not_handed_the_older_generations_handle(_tracing: &Tracing) { let tempdir = tempfile::TempDir::new().unwrap(); @@ -8846,3 +8993,49 @@ async fn aborting_a_transfer_waits_for_the_prefix_drop_it_handed_to_the_primary( "the abort returned before the primary finished dropping the transferred prefix" ); } + +/// `try_abort_transfer` can land between `append_target` and `drop_source_prefix` (see +/// `BackgroundTransfer::run`'s doc comment): the chunk this test appends models one that already +/// reached the archive when that happened. The real owner's next transfer would start from the +/// same, never-trimmed source range and derive the identical chunk id and bytes - exercised here +/// directly against the archive rather than by racing a real abort, since the archive is what +/// must tolerate the repeat (F19). +#[test] +async fn compressed_archive_append_reconciles_a_resumed_transfers_repeat_chunk(_tracing: &Tracing) { + let indexed_storage = Arc::new(InMemoryIndexedStorage::new()); + let archive_service = + CompressedOplogArchiveService::new(indexed_storage.clone(), 1, RetryConfig::default()); + let owned_agent_id = OwnedAgentId::new( + EnvironmentId::new(), + &AgentId { + component_id: ComponentId::new(), + agent_id: "resumed-transfer".into(), + }, + ); + let archive = archive_service + .open_fresh(&owned_agent_id, AgentMode::Durable) + .await; + + let chunk = vec![ + (OplogIndex::from_u64(1), OplogEntry::suspend().rounded()), + (OplogIndex::from_u64(2), OplogEntry::exited().rounded()), + ]; + archive.append(&chunk).await; + assert_eq!(archive.length().await, 1); + + // The resumed transfer's repeat: identical id, identical bytes. + archive.append(&chunk).await; + assert_eq!( + archive.length().await, + 1, + "a resumed transfer's identical repeat chunk must not duplicate" + ); + + // A different chunk landing at the same id is not explainable as a replay and must stay + // fatal rather than being papered over. + let different_chunk = vec![ + (OplogIndex::from_u64(1), OplogEntry::suspend().rounded()), + (OplogIndex::from_u64(2), OplogEntry::suspend().rounded()), + ]; + assert_panics(archive.append(&different_chunk)).await; +} diff --git a/golem-worker-executor/src/services/shard_manager.rs b/golem-worker-executor/src/services/shard_manager.rs index 8fe323237b..d1e99d9ceb 100644 --- a/golem-worker-executor/src/services/shard_manager.rs +++ b/golem-worker-executor/src/services/shard_manager.rs @@ -22,7 +22,7 @@ use golem_common::model::{ }; use golem_service_base::clients::shard_manager::{ShardLeaseError, ShardManagerError}; use std::collections::{BTreeMap, HashMap}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; use std::sync::{Arc, RwLock, Weak}; use std::time::{Duration, Instant}; use tracing::{error, info, warn}; @@ -124,6 +124,106 @@ async fn sleep_or_park(delay: RenewalDelay) { } } +/// Coordinates the announcements made from the renewal loop, so a slow one cannot stall lease +/// renewal. +/// +/// Relinquishing a still-loading agent waits for its whole component load and replay with no +/// timeout, so a set-changing renewal that awaited [`GrpcShardManagerService::announce_assignment_changed`] +/// inline would send no further renewal RPCs until that sweep finished, and the lease lapses for +/// the whole executor - the bug this exists to fix. At most one announcement runs at a time; a +/// request that arrives while one is already running is coalesced into exactly one more run after +/// it finishes, so nothing requested is ever dropped, and a loop that keeps correcting the set +/// does not pile up concurrent sweeps racing each other over the same agents. +/// +/// Deliberately not tracked by [`Shutdown::spawn`]: a sweep can run for as long as the slowest +/// agent's replay, and the renewal loop's own deregister - which *is* tracked, so `main` waits for +/// it - has to still land inside the shutdown grace regardless. Tracking this task the same way +/// would let a stale sweep hold the process open past that grace for a recovery nobody is waiting +/// on any more; a bare `tokio::spawn` is simply cut off at its next await point when the runtime +/// is dropped, which is the right fate for best-effort work like this one. +struct AnnouncementSingleFlight { + state: AtomicU8, +} + +impl AnnouncementSingleFlight { + /// No task running. + const IDLE: u8 = 0; + /// A task is running the announcement. + const RUNNING: u8 = 1; + /// A task is running, and a request arrived while it did; it reruns once more before going + /// idle. + const RUNNING_PENDING: u8 = 2; + + fn new() -> Self { + Self { + state: AtomicU8::new(Self::IDLE), + } + } + + /// Requests one announcement, without waiting for it to run. + /// + /// Spawns the task that runs it only when none is already running; a request that arrives + /// mid-run instead marks the running task to loop once more, so a burst of requests while a + /// sweep is in flight still produces at most one extra run after it. + fn request(svc: &Arc) { + loop { + let state = &svc.announcement_single_flight.state; + match state.compare_exchange( + Self::IDLE, + Self::RUNNING, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => { + let svc = svc.clone(); + tokio::spawn(Self::run(svc)); + return; + } + Err(Self::RUNNING) => { + match state.compare_exchange( + Self::RUNNING, + Self::RUNNING_PENDING, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return, + // The running task raced this and just went idle (or another requester + // already coalesced): re-read the state and try again from scratch. + Err(_) => continue, + } + } + // Already coalesced: a rerun is guaranteed without this request doing anything. + Err(_) => return, + } + } + } + + /// The task body [`Self::request`] spawns: runs the announcement, then either goes idle or, + /// if a request was coalesced while it ran, runs it once more. + async fn run(svc: Arc) { + loop { + svc.announce_assignment_changed().await; + let state = &svc.announcement_single_flight.state; + match state.compare_exchange( + Self::RUNNING, + Self::IDLE, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return, + Err(Self::RUNNING_PENDING) => { + state.store(Self::RUNNING, Ordering::SeqCst); + continue; + } + Err(other) => unreachable!( + "AnnouncementSingleFlight in state {other}, which only `request` and `run` \ + touch and neither ever stores it" + ), + } + } + } +} + pub struct GrpcShardManagerService { client: Arc, shard_service: Arc, @@ -173,6 +273,9 @@ pub struct GrpcShardManagerService { /// Source of tickets; never reused, so an attempt that outlives a full clear-and-defer cycle /// cannot collide with a later one. recovery_tickets: AtomicU64, + /// Coordinates announcements requested from the renewal loop; see + /// [`AnnouncementSingleFlight`]. + announcement_single_flight: AnnouncementSingleFlight, } impl GrpcShardManagerService { @@ -212,6 +315,7 @@ impl GrpcShardManagerService { rpc_deadline_floor, recovery_outstanding: AtomicU64::new(0), recovery_tickets: AtomicU64::new(0), + announcement_single_flight: AnnouncementSingleFlight::new(), }) } @@ -340,23 +444,39 @@ impl GrpcShardManagerService { break; } }; - // Raced against the token as well, not just the sleep before it: a renewal can - // be followed by a re-registration and a full agent recovery, and a termination - // signal arriving meanwhile has to be seen inside the grace `main` waits, which + // Raced against the token as well, not just the sleep before it: a termination + // signal arriving mid-renewal has to be seen inside the grace `main` waits, which // is the window the deregister below must be sent in. Abandoning the renewal // costs nothing: the process is stopping and handing the lease back. + // + // The renewal itself never runs the announcement inline any more - see + // `renew_shard_lease_internal` and `AnnouncementSingleFlight` - specifically so + // that an unbounded sweep following a set change or a re-registration can never + // be what the token is raced against here: this arm always returns promptly, and + // the loop keeps renewing while a sweep runs in its own task. tokio::select! { _ = shutdown_token.cancelled() => { svc.deregister().await; break; } - delay = svc.renew_shard_lease() => renewal_delay = delay, + (delay, announcement_owed) = svc.renew_shard_lease_internal() => { + renewal_delay = delay; + if announcement_owed { + AnnouncementSingleFlight::request(&svc); + } + } } } }); } - /// Applies a granted lease and returns the cadence for the next pass. + /// Applies a granted lease and returns the cadence for the next pass, together with whether + /// [`Self::announce_assignment_changed`] is owed - the caller decides how to run it, rather + /// than this awaiting it inline, so that a caller which must not block on a slow sweep (the + /// renewal loop, via `renew_shard_lease_internal`) can hand it to + /// [`AnnouncementSingleFlight`] instead. [`ShardManagerService::renew_shard_lease`] awaits it + /// right here for every other caller, so nothing outside the loop observes any change: a test + /// calling it directly still sees the hook run before it returns. /// /// The grant is the shard manager's set for this executor. Normally that /// is exactly what was claimed, and only the lease clock moves. When it is @@ -366,40 +486,49 @@ impl GrpcShardManagerService { /// set is older than the last delivery applied crossed with a push on the /// network: its set is ignored, or it would put the older set back, and /// its lease is adopted, because it answers this executor's own request. - async fn adopt_lease( + fn adopt_lease( &self, shard_epochs: BTreeMap, expires_at: Instant, revision: ShardLeaseRevision, - ) -> RenewalDelay { + ) -> (RenewalDelay, bool) { let shard_epochs: HashMap = shard_epochs.into_iter().collect(); - match self - .shard_service - .update_lease(&shard_epochs, expires_at, revision) - { + let announcement_owed = match self.shard_service.update_lease( + &shard_epochs, + expires_at, + revision, + ) { Ok(ShardDeliveryOutcome::Applied { set_changed: true }) => { info!( %revision, "Shard lease renewal corrected the shard set; sweeping and recovering agents" ); - self.announce_assignment_changed().await + true } Ok(ShardDeliveryOutcome::Applied { set_changed: false }) => { if self.recovery_outstanding.load(Ordering::SeqCst) != 0 { info!(%revision, "Running the agent recovery still owed from an earlier delivery"); - self.announce_assignment_changed().await + true + } else { + false } } - Ok(ShardDeliveryOutcome::Stale { delivered, applied }) => warn!( - %delivered, - %applied, - "Ignoring the shard set of a renewal older than the last delivery applied; the lease clock moved" - ), - Err(error) => warn!(%error, "Failed to apply a renewed shard lease"), - } + Ok(ShardDeliveryOutcome::Stale { delivered, applied }) => { + warn!( + %delivered, + %applied, + "Ignoring the shard set of a renewal older than the last delivery applied; the lease clock moved" + ); + false + } + Err(error) => { + warn!(%error, "Failed to apply a renewed shard lease"); + false + } + }; let cadence = renewal_interval_for(Some(expires_at), Instant::now()); self.record_granted(cadence); - cadence + (cadence, announcement_owed) } /// [`ShardManagerService::register`], carrying `previous_claim` to the shard manager: the set @@ -470,43 +599,20 @@ impl GrpcShardManagerService { } carried.clone() } -} -/// `(expires_at - now) / 3`, floored, so three attempts fit inside one lease. -/// -/// A lease that never expires yields `None`, which parks the -/// renewal loop instead of polling it — there is nothing to renew, and a -/// polling loop would be one wasted RPC per second per executor. -fn renewal_interval_for(expires_at: Option, now: Instant) -> RenewalDelay { - let expires_at = expires_at?; - Some(shard_lease::renewal_interval( - expires_at.saturating_duration_since(now), - )) -} - -#[async_trait] -impl ShardManagerService for GrpcShardManagerService { - /// Yes: a real shard manager moves shards between executors, so two of them can believe they - /// own the same agent at once and only the storage can tell them apart. - fn requires_oplog_fencing(&self) -> bool { - true - } - - async fn register( - &self, - port: u16, - pod_name: Option, - ) -> Result { - self.register_with_previous_claim(port, pod_name, BTreeMap::new()) - .await - } - - async fn renew_shard_lease(&self) -> RenewalDelay { + /// [`ShardManagerService::renew_shard_lease`], minus running the announcement it can end up + /// owing: returns the delay for the next pass and whether + /// [`Self::announce_assignment_changed`] still needs to run. Every caller other than the + /// renewal loop goes through the trait method, which awaits it right there - this split exists + /// solely so the loop (`start_renewal_loop`) can hand the announcement to + /// [`AnnouncementSingleFlight`] instead of awaiting it inline, so a slow sweep cannot stall the + /// next renewal RPC. + async fn renew_shard_lease_internal(&self) -> (RenewalDelay, bool) { let claim = match self.shard_service.current_assignment() { Ok(assignment) => assignment.claim(), Err(error) => { warn!(%error, "Skipping shard lease renewal, no shard assignment yet"); - return self.next_retry_delay(); + return (self.next_retry_delay(), false); } }; @@ -535,7 +641,7 @@ impl ShardManagerService for GrpcShardManagerService { deadline_ms = deadline.as_millis(), "Shard lease renewal did not answer in time; backing off" ); - return self.next_retry_delay(); + return (self.next_retry_delay(), false); } }; match renewed { @@ -548,7 +654,6 @@ impl ShardManagerService for GrpcShardManagerService { // the renewal was in flight goes with the next one. self.shard_service.retire_fence_learned_epochs(&fenced); self.adopt_lease(lease.shard_epochs, lease.expires_at, lease.revision) - .await } Err(ShardLeaseError::LeaseNotFound(details)) => { // The manager no longer knows this executor. Drop every shard @@ -575,7 +680,7 @@ impl ShardManagerService for GrpcShardManagerService { match registration { None => { error!("Cannot re-register: this executor never completed a registration"); - self.next_retry_delay() + (self.next_retry_delay(), false) } Some((port, pod_name)) => match self .register_with_previous_claim(port, pod_name, previous_claim) @@ -594,13 +699,14 @@ impl ShardManagerService for GrpcShardManagerService { "Re-registered with the shard manager after a lost lease" ); // The same announcement the initial - // registration and `assign_shards_internal` make. - self.announce_assignment_changed().await; - renewal_interval_for(assignment.expires_at, Instant::now()) + // registration and `assign_shards_internal` make - owed to the + // caller, exactly like a set-changing grant. + let delay = renewal_interval_for(assignment.expires_at, Instant::now()); + (delay, true) } Err(error) => { warn!(%error, "Re-registration after a lost lease failed"); - self.next_retry_delay() + (self.next_retry_delay(), false) } }, } @@ -610,10 +716,52 @@ impl ShardManagerService for GrpcShardManagerService { // runs down on its own and the self-fence starts refusing // admission the moment it passes. warn!(%error, "Shard lease renewal failed, retrying"); - self.next_retry_delay() + (self.next_retry_delay(), false) } } } +} + +/// `(expires_at - now) / 3`, floored, so three attempts fit inside one lease. +/// +/// A lease that never expires yields `None`, which parks the +/// renewal loop instead of polling it — there is nothing to renew, and a +/// polling loop would be one wasted RPC per second per executor. +fn renewal_interval_for(expires_at: Option, now: Instant) -> RenewalDelay { + let expires_at = expires_at?; + Some(shard_lease::renewal_interval( + expires_at.saturating_duration_since(now), + )) +} + +#[async_trait] +impl ShardManagerService for GrpcShardManagerService { + /// Yes: a real shard manager moves shards between executors, so two of them can believe they + /// own the same agent at once and only the storage can tell them apart. + fn requires_oplog_fencing(&self) -> bool { + true + } + + async fn register( + &self, + port: u16, + pod_name: Option, + ) -> Result { + self.register_with_previous_claim(port, pod_name, BTreeMap::new()) + .await + } + + /// Drives [`Self::renew_shard_lease_internal`] and, unlike the renewal loop, awaits the + /// announcement it may come back owing right here - so every caller other than the loop + /// itself (every test in this file included) sees exactly the synchronous behaviour this had + /// before the split: the hook has run by the time this returns. + async fn renew_shard_lease(&self) -> RenewalDelay { + let (delay, announcement_owed) = self.renew_shard_lease_internal().await; + if announcement_owed { + self.announce_assignment_changed().await; + } + delay + } async fn deregister(&self) { let claim = self @@ -1171,6 +1319,184 @@ mod tests { ); } + #[test] + // F29: relinquishing a still-loading agent waits for its whole component load and replay, + // with no timeout. If the renewal loop awaited the assignment-changed hook inline, that sweep + // would hold every later renewal RPC back and the lease would lapse for the whole executor + // while a single agent is loading. The hook here blocks on a flag the test controls, standing + // in for that sweep, and the assertion is that the loop keeps renewing right through it. + async fn a_slow_assignment_changed_hook_does_not_stall_the_renewal_loop() { + // Short enough that a stalled loop would be obvious within the test's own timeout, but + // the cadence this derives floors at `MIN_RENEWAL_INTERVAL` (1 s) regardless - see + // `renewal_interval` - so there is no point going below that. + let ttl = Duration::from_secs(3); + let mock = Arc::new( + MockShardManager::new() + .with_register(move |_| Ok(registration(Instant::now() + ttl, [(0, 1)]))) + .with_renew(move |_, _claimed| { + Ok(ShardLease { + // Always the widened set: the first grant is a real assignment change, + // and every later one only echoes it back (it now matches what was + // claimed), so only the first renewal ever owes an announcement. + shard_epochs: claim([(0, 1), (1, 1)]), + expires_at: Instant::now() + ttl, + revision: ShardLeaseRevision(1), + }) + }), + ); + let (service, shard_service) = make_service(mock.clone(), Shutdown::new()); + + let hook_calls = Arc::new(AtomicUsize::new(0)); + let released = Arc::new(AtomicBool::new(false)); + let calls = hook_calls.clone(); + let hook_released = released.clone(); + let hook: ShardAssignmentChangedHook = Arc::new(move || { + let calls = calls.clone(); + let released = hook_released.clone(); + Box::pin(async move { + while !released.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + calls.fetch_add(1, Ordering::SeqCst); + Ok(RecoveryOutcome::Recovered) + }) + }); + service.set_assignment_changed_hook(&hook); + + let assignment = service.register(PORT, None).await.unwrap(); + shard_service.register( + assignment.number_of_shards, + &assignment.shard_epochs, + assignment.expires_at, + assignment.revision, + ); + + // The first renewal widens the set, so its announcement enters the hook and blocks + // there - on its own task, not on the loop. + for _ in 0..200 { + if !mock.renew_calls().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + let renewals_before = mock.renew_calls().len(); + assert!( + renewals_before >= 1, + "the loop must have issued the first renewal" + ); + + // Several renewal periods pass (cadence is ~1s) while the hook stays blocked: the loop + // must keep renewing rather than waiting on the sweep, which is exactly the stall F29 + // fixes. Each of these renewals also finds `recovery_outstanding` still set by the first + // one - the sweep has not reported back yet - and asks for the announcement again; that + // is correct (an unchanged grant re-runs a recovery still owed), and the single-flight + // coalesces every one of these requests into exactly one rerun, asserted below. + for _ in 0..100 { + if mock.renew_calls().len() >= renewals_before + 3 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + let renewals_after = mock.renew_calls().len(); + assert!( + renewals_after >= renewals_before + 3, + "further renewals must happen while the hook is blocked; before={renewals_before} \ + after={renewals_after}" + ); + assert_eq!( + hook_calls.load(Ordering::SeqCst), + 0, + "the hook must still be blocked at this point" + ); + + // Releasing it lets the sweep complete - and, deterministically, exactly one coalesced + // rerun right after: at least one of the renewals just observed above landed while the + // single-flight was already running, which is what put it in its "one more owed" state. + released.store(true, Ordering::SeqCst); + for _ in 0..200 { + if hook_calls.load(Ordering::SeqCst) >= 2 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!( + hook_calls.load(Ordering::SeqCst), + 2, + "the blocked sweep and exactly one coalesced rerun, never one rerun per renewal" + ); + } + + #[test] + // The single-flight coordinator behind the fix above must not let a burst of requests that + // arrive while one announcement is already running turn into one run per request: exactly one + // more run is owed, coalescing the rest. Drives `AnnouncementSingleFlight` directly, bypassing + // the renewal loop entirely, since that is the unit actually being tested here. + async fn concurrent_announcement_requests_coalesce_into_one_more_run() { + let mock = Arc::new(MockShardManager::new()); + let (service, _shard_service) = make_service(mock, Shutdown::new()); + + let calls = Arc::new(AtomicUsize::new(0)); + let started = Arc::new(AtomicUsize::new(0)); + let released = Arc::new(AtomicBool::new(false)); + let hook_calls = calls.clone(); + let hook_started = started.clone(); + let hook_released = released.clone(); + let hook: ShardAssignmentChangedHook = Arc::new(move || { + let calls = hook_calls.clone(); + let started = hook_started.clone(); + let released = hook_released.clone(); + Box::pin(async move { + // Only the first call blocks; the coalesced rerun this test looks for must be + // observable without releasing a second time. + if started.fetch_add(1, Ordering::SeqCst) == 0 { + while !released.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + calls.fetch_add(1, Ordering::SeqCst); + Ok(RecoveryOutcome::Recovered) + }) + }); + service.set_assignment_changed_hook(&hook); + + AnnouncementSingleFlight::request(&service); + // Wait for the run to actually start (and block) before piling more requests onto it. + for _ in 0..200 { + if started.load(Ordering::SeqCst) >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(started.load(Ordering::SeqCst), 1); + + // A burst of requests while the first run is still blocked. + for _ in 0..5 { + AnnouncementSingleFlight::request(&service); + } + + released.store(true, Ordering::SeqCst); + + for _ in 0..200 { + if calls.load(Ordering::SeqCst) >= 2 { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a burst of requests while one run is in flight must coalesce into exactly one \ + more run" + ); + // Give a wrongly-spawned extra run a chance to show up before declaring victory. + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + started.load(Ordering::SeqCst), + 2, + "exactly two runs total: the first, and the one coalesced burst" + ); + } + #[test] // The deregister has a deadline of its own, sized against the grace `main` waits rather than // against the lease, so a shard manager that accepts the call and never answers cannot hold diff --git a/golem-worker-executor/src/services/worker_fork.rs b/golem-worker-executor/src/services/worker_fork.rs index 0ad1925d46..6d4ee2b917 100644 --- a/golem-worker-executor/src/services/worker_fork.rs +++ b/golem-worker-executor/src/services/worker_fork.rs @@ -20,7 +20,7 @@ use crate::durable_host::websocket::WebSocketConnectionPool; use crate::metrics::workers::record_worker_call; use crate::model::ExecutionStatus; use crate::services::events::Events; -use crate::services::oplog::plugin::OplogProcessorPlugin; +use crate::services::oplog::plugin::{OplogProcessorPlugin, try_join_background_work}; use crate::services::oplog::{CommitLevel, MultiLayerOplog, Oplog, OplogOps}; use crate::services::resource_limits::ResourceLimits; use crate::services::rpc::Rpc; @@ -846,15 +846,25 @@ fn rewrite_forked_oplog_entry( /// Commits the fork target's copied oplog and closes its handle, before the target is resumed. /// /// The handle asserts no epoch, so nothing it started may still be writing once the target's -/// owner opens the oplog at its own epoch. Dropping it does not ensure that: when the copy reaches -/// the entry count limit, the final commit schedules an archive transfer that holds its own -/// reference to the handle and ends by dropping the primary's prefix, and deleting the primary -/// oplog once that empties it, over whatever the owner appended meanwhile. The transfer is ended -/// through the handle, not the agent-keyed transfer registry, which the owner's open overwrites. +/// owner opens the oplog at its own epoch. Dropping it does not ensure that on its own: +/// - When an oplog processor plugin is configured, the handle is a `ForwardingOplog` running its +/// own actor and periodic-commit timer in the background. Dropping only aborts them; a +/// checkpoint commit already under way when the caller moves on would still land, unfenced, +/// after the owner has opened its own primary. `try_join_background_work` stops the timer and +/// waits for the actor to drain everything already queued - including a tick the timer sent in +/// the instant before it was stopped - before this function returns. +/// - When the copy reaches the entry count limit, a commit (the explicit one below, or one the +/// forwarding actor ran while draining) schedules an archive transfer that holds its own +/// reference to the handle and ends by dropping the primary's prefix, and deleting the primary +/// oplog once that empties it, over whatever the owner appended meanwhile. `try_abort_transfer` +/// ends that transfer through the handle, not the agent-keyed transfer registry, which the +/// owner's open overwrites - and runs after the forwarding actor has been joined, so it also +/// catches a transfer the drained actor only just started. pub(crate) async fn close_fork_target_oplog( new_oplog: Arc, ) -> Result<(), WorkerExecutorError> { new_oplog.commit(CommitLevel::Always).await?; + try_join_background_work(&new_oplog).await; MultiLayerOplog::try_abort_transfer(&new_oplog).await; drop(new_oplog); Ok(()) diff --git a/golem-worker-executor/src/storage/indexed/sqlite.rs b/golem-worker-executor/src/storage/indexed/sqlite.rs index 24dac0ec42..42e6accc35 100644 --- a/golem-worker-executor/src/storage/indexed/sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/sqlite.rs @@ -104,6 +104,19 @@ impl SqliteIndexedStorage { } } + /// sqlx has no `Encode` for `u64`, so a value that must stay integer-bound (rather + /// than go through `Json`, which encodes as TEXT - see [`Self::upsert_oplog_metadata`]) has to + /// cross to `i64` first. Checked, like Postgres's own `to_i64`: an unchecked `as i64` on a + /// value above `i64::MAX` wraps to negative, and reading that back `as u64` produces a + /// spuriously huge epoch instead of failing loudly. + fn to_i64(value: u64, field_name: &'static str) -> Result { + i64::try_from(value).map_err(|_| { + IndexedStorageError::Other(format!( + "SQLite indexed storage cannot represent {field_name}={value} as i64" + )) + }) + } + fn classify_repo_error(err: RepoError) -> IndexedStorageError { if err.is_transient() { IndexedStorageError::Transient(err.to_string()) @@ -348,8 +361,9 @@ impl IndexedStorage for SqliteIndexedStorage { let namespace = Self::namespace(namespace); // `i64`, not `u64`: sqlx has no `Encode` for `u64`, which is why ids elsewhere in // this file go through `Json`. That encodes as TEXT, and comparison affinity is applied - // per operand, so this column stays integer-bound everywhere. - let epoch = shard_epoch.0 as i64; + // per operand, so this column stays integer-bound everywhere. Checked (see `to_i64`) + // rather than `as i64`, which would silently wrap an out-of-range epoch to negative. + let epoch = Self::to_i64(shard_epoch.0, "shard_epoch")?; let mut api = self.pool.with_rw(svc_name, api_name); let result = api @@ -696,4 +710,64 @@ mod tests { vec![(2, b"existing".to_vec())] ); } + + #[test] + // The column is `i64`-bound (see `to_i64`'s doc). An epoch that does not fit it must be + // rejected here rather than silently wrapped to a negative value that a later `epoch as u64` + // read turns into a spuriously huge one - the class of bug that let a corrupted epoch panic + // downstream in the shard manager (`ShardEpoch::next`'s `checked_add(1).expect(..)`). + async fn upsert_oplog_metadata_rejects_an_epoch_that_does_not_fit_i64() { + let tempdir = tempfile::tempdir().unwrap(); + let storage = sqlite_storage( + tempdir + .path() + .join("indexed.db") + .to_string_lossy() + .into_owned(), + ) + .await; + let namespace = oplog_namespace("sqlite-epoch-overflow"); + + let result = storage + .upsert_oplog_metadata( + "test", + "upsert_oplog_metadata", + namespace, + "oplog", + ShardEpoch(u64::MAX), + ) + .await; + + assert!( + matches!(result, Err(IndexedStorageError::Other(_))), + "an epoch above i64::MAX must be a rejected write, not a wrapped negative one, got {result:?}" + ); + } + + #[test] + // The largest value that does fit is the boundary right below the rejected one, and must + // still succeed - a regression here would mean the checked conversion rejects valid input. + async fn upsert_oplog_metadata_accepts_the_largest_epoch_that_fits_i64() { + let tempdir = tempfile::tempdir().unwrap(); + let storage = sqlite_storage( + tempdir + .path() + .join("indexed.db") + .to_string_lossy() + .into_owned(), + ) + .await; + let namespace = oplog_namespace("sqlite-epoch-boundary"); + + storage + .upsert_oplog_metadata( + "test", + "upsert_oplog_metadata", + namespace, + "oplog", + ShardEpoch(i64::MAX as u64), + ) + .await + .unwrap(); + } } diff --git a/golem-worker-executor/src/worker/instance.rs b/golem-worker-executor/src/worker/instance.rs index 40b10ea9f1..d6864ae3f8 100644 --- a/golem-worker-executor/src/worker/instance.rs +++ b/golem-worker-executor/src/worker/instance.rs @@ -21,7 +21,7 @@ use crate::durable_host::tool::operation::{DeferredAdmissionTable, OwnerToolOper use crate::model::ExecutionStatus; use crate::services::active_agents::WorkerComponentCharge; use crate::services::agent_filesystem::FilesystemGenerationHandle; -use crate::services::oplog::{CommitLevel, Oplog, OplogError}; +use crate::services::oplog::{CommitLevel, Oplog, OplogError, OplogFence}; use crate::services::resource_limits::AtomicResourceEntry; use crate::services::{HasActiveAgents, HasComponentService, HasWasmtimeEngine}; use crate::workerctx::WorkerCtx; @@ -420,8 +420,11 @@ impl OwnerExecution { } } - pub async fn commit(&self, level: CommitLevel) -> OplogIndex { - self.commit.commit_and_update_state(level).await.0 + pub async fn commit(&self, level: CommitLevel) -> Result { + self.commit + .commit_and_update_state(level) + .await + .map(|(index, _)| index) } } @@ -723,7 +726,7 @@ impl InstanceHost { // process crash can replay and persist the same instantiation growth again. owner .add_and_commit_oplog(OplogEntry::grow_memory(live_instantiation_growth)) - .await; + .await?; owner .startup_linear_memory_bytes .store(allocated_bytes, Ordering::Release); diff --git a/golem-worker-executor/src/worker/invocation_loop.rs b/golem-worker-executor/src/worker/invocation_loop.rs index a4e255940d..6c29902a0a 100644 --- a/golem-worker-executor/src/worker/invocation_loop.rs +++ b/golem-worker-executor/src/worker/invocation_loop.rs @@ -252,6 +252,13 @@ impl InvocationLoop { 'outer: loop { self.release_terminal_interrupt().await; + // Never a new instance for an agent given up here, whichever path led back to this + // point: it would reopen the oplog at an epoch this executor no longer holds. + if self.parent.is_relinquished() { + self.release_concurrent_agent_permit(); + self.stop_startup_given_up().await; + break; + } // ADMISSION: gates the start of a generation, so // fencing refuses new generations and never interrupts a running one. if let Err(error) = self.parent.shard_service().check_admission(&agent_id) { @@ -288,9 +295,15 @@ impl InvocationLoop { continue; } InterruptKind::Suspend(ts) => { - self.parent + if self + .parent .add_and_commit_oplog(OplogEntry::suspend()) - .await; + .await + .is_err() + { + self.stop_startup_given_up().await; + break; + } if ts < *self.parent.last_resume_request.lock().await { debug!( "Suspend during instantiation ignored because there was a resume request since it" @@ -303,9 +316,15 @@ impl InvocationLoop { } } InterruptKind::Interrupt(_) => { - self.parent + if self + .parent .add_and_commit_oplog(OplogEntry::interrupted()) - .await; + .await + .is_err() + { + self.stop_startup_given_up().await; + break; + } self.parent.complete_startup( self.start_attempt, Err(WorkerExecutorError::Interrupted { kind }), @@ -316,19 +335,7 @@ impl InvocationLoop { InterruptKind::ShardLost => { // Nothing is written: the oplog belongs to the shard's new owner // now. Whoever was waiting for this start is told to look there. - // - // A fence found by a host call during instantiation arrives here - // without `relinquish()` having run, so the agent is marked given up - // now: the stop below then tears its entity bodies down as - // `ShardLost`, fails its waiters and removes only this generation. A - // reason `relinquish()` already recorded is kept. - self.parent - .mark_relinquished(RelinquishReason::Fenced(None)); - self.parent.complete_startup( - self.start_attempt, - Err(WorkerExecutorError::ShardingNotReady), - ); - self.stop_unloaded(None).await; + self.stop_startup_given_up().await; break; } } @@ -484,7 +491,10 @@ impl InvocationLoop { &self.filesystem_activity, ), || async { - self.parent + // A refusal marks the agent given up, which the stop + // below acts on; there is nothing else to undo. + let _ = self + .parent .add_and_commit_oplog(OplogEntry::suspend()) .await; }, @@ -593,6 +603,21 @@ impl InvocationLoop { break; } + // Whatever was decided, an agent given up here is not restarted, retried later or + // parked for a resume on this executor: the shard's new owner resumes it. + if self.parent.is_relinquished() { + debug!( + %agent_id, + ?final_decision, + "Invocation queue loop stopping an agent this executor has given up" + ); + self.stop_startup_given_up().await; + if cleanup_ephemeral_worker { + self.archive_ephemeral_oplog(); + } + break; + } + match final_decision { None | Some(RetryDecision::None) => { debug!( @@ -667,16 +692,31 @@ impl InvocationLoop { .await .current_idempotency_key .clone(); - match kind { + // Given up, before or by this interrupt: the oplog is + // the new owner's to write, so no lifecycle entry and no + // failure is recorded for an invocation it runs. + if matches!(kind, InterruptKind::ShardLost) + || self.parent.is_relinquished() + { + self.stop_startup_given_up().await; + break 'outer; + } + let recorded = match kind { InterruptKind::Suspend(_) => { - self.parent.add_and_commit_oplog(OplogEntry::suspend()).await; + self.parent.add_and_commit_oplog(OplogEntry::suspend()).await.map(|_| ()) } InterruptKind::Interrupt(_) => { - self.parent.add_and_commit_oplog(OplogEntry::interrupted()).await; + self.parent.add_and_commit_oplog(OplogEntry::interrupted()).await.map(|_| ()) } - InterruptKind::Restart | InterruptKind::Jump => {} - // The oplog is the new owner's to write. - InterruptKind::ShardLost => {} + InterruptKind::Restart + | InterruptKind::Jump + | InterruptKind::ShardLost => Ok(()), + }; + // Refused, the agent has been given up: nothing restarts + // in place. + if recorded.is_err() { + self.stop_startup_given_up().await; + break 'outer; } if matches!(kind, InterruptKind::Interrupt(_)) && let Some(key) = current_idempotency_key @@ -790,7 +830,28 @@ impl InvocationLoop { self.permit_state.release(); } + /// Stops a generation this executor has given up, because its shard was lost or its oplog + /// refused a lifecycle entry: the waiters are told to look for the shard's new owner. + /// + /// A fence found by a host call during instantiation arrives without `relinquish()` having + /// run, so the agent is marked given up here: the stop then tears its entity bodies down as + /// `ShardLost`, fails its waiters and removes only this generation. A reason already recorded + /// is kept. + async fn stop_startup_given_up(&self) { + self.parent + .mark_relinquished(RelinquishReason::Fenced(None)); + self.stop_unloaded(None).await; + } + async fn stop_unloaded(&self, startup_failure: Option) { + // A generation this executor has given up keeps the retry answer as its startup failure, + // like `Worker::relinquish` does, whichever exit stopped it: otherwise a readiness waiter + // resolved by the stop, or a handle kept past this generation, is told it may proceed. + let startup_failure = if self.parent.is_relinquished() { + Some(self.parent.relinquish_error()) + } else { + startup_failure + }; self.parent.complete_startup( self.start_attempt, Err(startup_failure.clone().unwrap_or_else(|| { @@ -960,6 +1021,13 @@ impl InvocationLoop { self.parent.record_recovery_failure(&err).await; err }; + // A generation given up keeps the error that sends its callers to the shard's + // new owner, whatever failed on the way out. + let err = if self.parent.is_relinquished() { + self.parent.relinquish_error() + } else { + err + }; self.parent .complete_startup(self.start_attempt, Err(err.clone())); let final_state = if filesystem_cleanup_failed { @@ -995,9 +1063,14 @@ impl InvocationLoop { // Making sure all pending commits are flushed // Make sure all pending commits are done let worker = store.lock().await.data().get_public_state().worker(); - worker + if worker .commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await + .is_err() + { + // Given up: its status is not this executor's to persist any more. + return; + } // The worker is going idle; persist its cached status synchronously now instead of leaving // it for the next background sweep, so reads of an idle worker see an up-to-date blob. @@ -1326,6 +1399,13 @@ impl InnerInvocationLoop<'_, Ctx> { break self.interrupt(interrupt).await; } + // Given up by a path that queues no interrupt, such as a write refused + // outside the guest. Nothing more is taken from the queue: the shard's + // new owner runs it. + if self.parent.is_relinquished() { + break CommandOutcome::BreakInnerLoop(RetryDecision::None); + } + let message = self.pop_ready_internal_invocation().await; let result = if let Some(message) = message { @@ -1560,6 +1640,9 @@ impl InnerInvocationLoop<'_, Ctx> { /// first pending_updates, then pending_invocations async fn drain_pending_from_status(&mut self) -> CommandOutcome { loop { + if self.parent.is_relinquished() { + break CommandOutcome::BreakInnerLoop(RetryDecision::None); + } let status = self.parent.get_non_detached_last_known_status().await; // First, try to process a pending update @@ -2024,6 +2107,11 @@ impl Invocation<'_, Ctx> { /// or a manual update request (which involves invoking the exported save-snapshot functions, so /// it is a special case of the exported function invocation). async fn external_invocation(&mut self, inner: TimestampedAgentInvocation) -> CommandOutcome { + // Rechecked here as well as where the invocation was taken: hydrating it and waiting for + // the store both leave room for the agent to be given up in between. + if self.parent.is_relinquished() { + return CommandOutcome::BreakInnerLoop(RetryDecision::None); + } match inner.invocation { AgentInvocation::ManualUpdate { target_revision } => { self.manual_update(target_revision).await @@ -2040,16 +2128,26 @@ impl Invocation<'_, Ctx> { debug!( "Skipping enqueued invocation with idempotency key {idempotency_key} as it already has a result" ); - if let Err(error) = - self.parent.cancel_invocation(idempotency_key.clone()).await + match self + .parent + .cancel_invocation_from_loop(idempotency_key.clone()) + .await { - warn!( - agent_id = %self.owned_agent_id.agent_id, - "Failed to remove completed invocation from the pending queue: {error}" - ); - return CommandOutcome::BreakInnerLoop(RetryDecision::Immediate); + Ok(true) => CommandOutcome::Continue, + // A stop is waiting for this loop to exit. + Ok(false) => CommandOutcome::BreakInnerLoop(RetryDecision::None), + Err(_) if self.parent.is_relinquished() => { + CommandOutcome::BreakInnerLoop(RetryDecision::None) + } + Err(error) => { + warn!( + agent_id = %self.owned_agent_id.agent_id, + %error, + "Failed to remove completed invocation from the pending queue" + ); + CommandOutcome::BreakInnerLoop(RetryDecision::Immediate) + } } - CommandOutcome::Continue } } else { self.invoke_agent(invocation).await @@ -2380,22 +2478,17 @@ impl Invocation<'_, Ctx> { .and_then(|result| result) { tracing::error!(%error, "Failed to complete durable streaming session"); - if let Some(reason) = - session_completion_relinquishment(&error, self.parent.oplog.fence()) - { - self.parent.mark_relinquished(reason); - let decision = self - .store + // An in-place retry would reopen the oplog at an epoch this executor no + // longer holds, so a lost shard gives the agent up instead. + if self.parent.relinquish_if_shard_lost(&error) { + self.store .data_mut() .on_invocation_failure( &full_function_name, &TrapType::Interrupt(InterruptKind::ShardLost), ) .await; - return failed_agent_invocation_outcome( - self.parent.agent_mode(), - decision, - ); + return CommandOutcome::BreakInnerLoop(RetryDecision::None); } return failed_agent_invocation_outcome( self.parent.agent_mode(), @@ -2415,6 +2508,9 @@ impl Invocation<'_, Ctx> { .data_mut() .on_invocation_failure(&full_function_name, &TrapType::Interrupt(kind)) .await; + if let Some(outcome) = self.given_up_outcome() { + return outcome; + } if self.uses_streams { let _ = self .parent @@ -2426,20 +2522,15 @@ impl Invocation<'_, Ctx> { // Intercepted before the arm below, which would flatten it into an // `AgentError::InternalError` and append an `Error` entry to the very oplog that // just refused the write. - Err(error @ WorkerExecutorError::OplogFenced { .. }) => { - let decision = self - .store + Err(WorkerExecutorError::OplogFenced { .. }) => { + self.store .data_mut() .on_invocation_failure( &full_function_name, &TrapType::Interrupt(InterruptKind::ShardLost), ) .await; - let _ = self - .parent - .fail_durable_streaming_session(idempotency_key, error.to_string()) - .await; - failed_agent_invocation_outcome(self.parent.agent_mode(), decision) + CommandOutcome::BreakInnerLoop(RetryDecision::None) } Err(error) => { // The success hook commits `AgentInvocationFinished`; if the storage refused that @@ -2460,6 +2551,9 @@ impl Invocation<'_, Ctx> { .data_mut() .on_invocation_failure(&full_function_name, &trap_type) .await; + if let Some(outcome) = self.given_up_outcome() { + return outcome; + } if self.uses_streams { let _ = self .parent @@ -2471,6 +2565,16 @@ impl Invocation<'_, Ctx> { } } + /// The outcome of an invocation that failed on an agent this executor has given up, `None` + /// while the agent is still its own. Checked after `on_invocation_failure`, which marks a lost + /// shard. Nothing more is written: a terminal streaming-session failure, like any other + /// terminal record, would end an invocation the shard's new owner resumes. + fn given_up_outcome(&self) -> Option { + self.parent + .is_relinquished() + .then_some(CommandOutcome::BreakInnerLoop(RetryDecision::None)) + } + /// The logic handling an agent invocation that did not succeed. async fn agent_invocation_failed( &mut self, @@ -2533,6 +2637,9 @@ impl Invocation<'_, Ctx> { } None => RetryDecision::None, }; + if let Some(outcome) = self.given_up_outcome() { + return outcome; + } if self.uses_streams && decision == RetryDecision::None { let _ = self @@ -2658,12 +2765,20 @@ impl Invocation<'_, Ctx> { .await { Ok(update_description) => { - // Enqueue the update - self.parent.enqueue_update(update_description).await; - - // Reactivate the worker - CommandOutcome::BreakInnerLoop(RetryDecision::Immediate) - // Stop processing the queue to avoid race conditions + // Refused, or the worker is stopping: nothing restarts it here, and the + // manual update stays pending for the next generation. + match self + .parent + .enqueue_update_from_loop(update_description) + .await + { + // Reactivate the worker; stop processing the queue to avoid race + // conditions + Ok(true) => CommandOutcome::BreakInnerLoop(RetryDecision::Immediate), + Ok(false) | Err(_) => { + CommandOutcome::BreakInnerLoop(RetryDecision::None) + } + } } Err(error) => { self.fail_update( @@ -2704,6 +2819,12 @@ impl Invocation<'_, Ctx> { .await } Ok(InvokeResult::Interrupted { interrupt_kind, .. }) => { + // Marked before `fail_update` checks the mark: a `ShardLost` interrupt is a lost + // shard whether or not anything marked the agent on its way here. + self.parent + .relinquish_if_shard_lost(&WorkerExecutorError::Interrupted { + kind: interrupt_kind, + }); self.fail_update( target_revision, format!("failed to get a snapshot for manual update: {interrupt_kind:?}"), @@ -2794,11 +2915,23 @@ impl Invocation<'_, Ctx> { target_revision: ComponentRevision, error: String, ) -> CommandOutcome { - self.store + // A `FailedUpdate` drops the pending manual update from the status, so written for an + // agent given up here it would keep the shard's new owner from ever applying it. Nothing + // is written; the update stays pending and runs there. + if self.parent.is_relinquished() { + return CommandOutcome::BreakInnerLoop(RetryDecision::None); + } + // Refused, the agent has been given up: it stops rather than carrying on at a revision + // whose failed update was never recorded. + match self + .store .data() .on_worker_update_failed(target_revision, Some(error)) - .await; - CommandOutcome::Continue + .await + { + Ok(()) => CommandOutcome::Continue, + Err(_) => CommandOutcome::BreakInnerLoop(RetryDecision::None), + } } /// Extends the invocation context with a new span containing information about the invocation @@ -2942,14 +3075,19 @@ impl Invocation<'_, Ctx> { .agent_wallet_cards_snapshot(); let wallet_generation = self.store.data().durable_ctx().wallet_generation(); - self.parent + if self + .parent .add_and_commit_oplog(OplogEntry::snapshot( payload, snapshot.mime_type, active_cards, wallet_generation, )) - .await; + .await + .is_err() + { + return CommandOutcome::BreakInnerLoop(RetryDecision::None); + } debug!("Periodic snapshot saved successfully"); // A snapshot is committed between invocations, so no jumpable @@ -3064,24 +3202,6 @@ fn failed_agent_invocation_outcome( } } -/// Why the agent is given up when its streaming session could not be completed, if it is. -/// -/// A commit the storage refused reaches the completion as `OplogFenced` or flattened into a -/// runtime error, and in both cases the oplog has latched the fence. An in-place retry would -/// reopen the oplog at the epoch that was just refused, so the agent is relinquished instead. -fn session_completion_relinquishment( - error: &WorkerExecutorError, - latched: Option, -) -> Option { - match latched { - Some(fence) => Some(RelinquishReason::Fenced(Some(Box::new(fence)))), - None if matches!(error, WorkerExecutorError::OplogFenced { .. }) => { - Some(RelinquishReason::Fenced(None)) - } - None => None, - } -} - fn should_cleanup_terminal_ephemeral_invocation( agent_mode: AgentMode, is_agent_component: bool, @@ -3144,9 +3264,9 @@ mod tests { close_usage_before_delete, coalesce_filesystem_limit_update, exit_owner_failure, failed_agent_invocation_outcome, finish_filesystem_limit_unload, periodic_snapshot_failure_outcome, publish_unload_outcome, run_invocation_loop_task, - session_completion_relinquishment, snapshot_action_at, snapshot_baseline_timestamp, - spawn_module_owned_unload, successful_agent_invocation_outcome, - unload_resident_agent_ownership, wait_for_resident_wakeup, + snapshot_action_at, snapshot_baseline_timestamp, spawn_module_owned_unload, + successful_agent_invocation_outcome, unload_resident_agent_ownership, + wait_for_resident_wakeup, }; use crate::durable_host::tool::operation::OwnerFailureWinner; use crate::sandbox_filesystem::ScriptedSandboxFilesystem; @@ -3161,8 +3281,8 @@ mod tests { use crate::worker::invocation::InvokeResult; use crate::worker::{ EvictionClass, FilesystemPressureEligibility, FinalWorkerState, - PendingLiveInvocationDisposition, RelinquishReason, RetryDecision, RunningAgent, - StoppingWorker, UnloadReason, WorkerCommand, WorkerInstance, complete_stopping_worker, + PendingLiveInvocationDisposition, RetryDecision, RunningAgent, StoppingWorker, + UnloadReason, WorkerCommand, WorkerInstance, complete_stopping_worker, }; use crate::workerctx::default::Context; use golem_common::model::AgentInvocationKind; @@ -3956,39 +4076,6 @@ mod tests { ); } - /// A session completion that failed on a fence must not take the in-place restart: that - /// would reopen the oplog at the epoch the storage just refused. - #[test] - fn a_fenced_session_completion_relinquishes_instead_of_restarting() { - let agent_id = golem_common::model::AgentId { - component_id: golem_common::model::component::ComponentId(uuid::Uuid::from_u128(1)), - agent_id: "fenced".to_string(), - }; - let fence = crate::services::oplog::OplogFence { - agent_id: agent_id.clone(), - expected_epoch: golem_common::model::ShardEpoch(3), - actual_epoch: Some(golem_common::model::ShardEpoch(4)), - }; - let flattened = WorkerExecutorError::runtime("durable stream commit failed"); - - assert!(matches!( - session_completion_relinquishment(&flattened, Some(fence.clone())), - Some(RelinquishReason::Fenced(Some(latched))) if *latched == fence - )); - assert!(matches!( - session_completion_relinquishment( - &WorkerExecutorError::OplogFenced { - agent_id, - expected_epoch: 3, - actual_epoch: Some(4), - }, - None, - ), - Some(RelinquishReason::Fenced(None)) - )); - assert!(session_completion_relinquishment(&flattened, None).is_none()); - } - #[test] fn durable_live_streaming_invocation_always_reconstructs_the_store() { assert_eq!( diff --git a/golem-worker-executor/src/worker/lifecycle.rs b/golem-worker-executor/src/worker/lifecycle.rs index 53bc8122a6..e6d6cb8876 100644 --- a/golem-worker-executor/src/worker/lifecycle.rs +++ b/golem-worker-executor/src/worker/lifecycle.rs @@ -433,7 +433,7 @@ impl Worker { debug!("Enqueuing update"); worker .enqueue_update(UpdateDescription::Automatic { target_revision }) - .await; + .await?; match decision { UpdateDecision::Queue => { diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index c37e7c6550..c3a9c5a84e 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -672,7 +672,8 @@ impl DurableStreamConsumerJournal for WorkerDurableStreamConsume let (_, changed) = self .state_actor .commit_and_update_state(CommitLevel::Always) - .await; + .await + .map_err(|fence| OplogError::Fenced(fence).to_string())?; if changed { self.state_actor.notify_status_changed(); } @@ -771,16 +772,18 @@ fn is_infrastructure_recovery_error(error: &WorkerExecutorError) -> bool { } } -/// Why the agent is given up instead of having its startup or replay failure persisted, if it is. +/// Why the agent is given up because of `error`, if the failure is really a lost shard. /// -/// A failure that is really a lost shard is not this executor's to record. The append would be -/// refused by the same fence that caused it, and on a shard given up without one it would write a -/// `Recovery` error into an oplog the new owner is already recovering, which would then replay a -/// failure this executor had no right to append. +/// Such a failure is not this executor's to record or to retry. An append would be refused by the +/// same fence that caused it, and on a shard given up without one it would write an error into an +/// oplog the new owner is already recovering. A retry in place would reopen the oplog at the epoch +/// this executor no longer holds. Every path that fails on a lost shard - startup and replay +/// recovery, streaming-session completion, the dropped-call drain, an interrupt - classifies with +/// this one predicate, so none of them can take the in-place retry another one refuses. /// /// `latched` is the fence the oplog holds, for a refusal that reached its caller flattened into /// some other error; the classified shapes are recognised without one. -fn recovery_failure_relinquishment( +pub(crate) fn shard_lost_relinquishment( error: &WorkerExecutorError, latched: Option, ) -> Option { @@ -848,6 +851,18 @@ impl Worker { self.relinquishment.get().is_some() } + /// Marks the agent given up when `error` means its shard was lost, per + /// [`shard_lost_relinquishment`]. Returns whether the agent is given up, by this failure or + /// for an earlier reason: either way the caller must neither record the failure nor retry in + /// place. Marked rather than stopped, for the same reason as [`Self::mark_relinquished`]: the + /// callers unwind to a stop, some of them while holding the instance lock. + pub(crate) fn relinquish_if_shard_lost(&self, error: &WorkerExecutorError) -> bool { + if let Some(reason) = shard_lost_relinquishment(error, self.oplog.fence()) { + self.mark_relinquished(reason); + } + self.is_relinquished() + } + /// Stops this worker through this handle, whichever generation it is. Nothing public reaches /// the stop through an arbitrary handle, and a handle kept past its generation is exactly what /// the tests using this need. @@ -1355,6 +1370,10 @@ impl Worker { { let init_idempotency_key = IdempotencyKey::new(format!("init-{}", worker.agent_id())); let init_input = agent_id.parameters.value().clone(); + // Returned rather than unwrapped: an agent created at an epoch another executor has + // already claimed is refused here with `OplogFenced`, which its caller retries on the + // shard's owner. Nothing is lost by returning early, since the next load repeats this + // check against the same short oplog. worker .enqueue_worker_invocation(AgentInvocation::AgentInitialization { idempotency_key: init_idempotency_key, @@ -1362,8 +1381,7 @@ impl Worker { invocation_context: invocation_context_stack.clone(), principal, }) - .await - .expect("Failed enqueuing initial agent invocations to worker"); + .await?; }; if Ctx::ALLOW_LIVE_REPAIR_OF_INCOMPLETE_DURABLE_CALLS && worker.has_durable_stream_history() @@ -1606,6 +1624,14 @@ impl Worker { oom_retry_count: u32, existing_start_attempt: Option, ) -> Result, WorkerExecutorError> { + // A handle kept past a generation this executor has given up must not start it again: the + // start would take permits, could append `Resumed` to an oplog the new owner now writes, + // and a failure in it would publish, by agent id, to the waiters of the generation that + // replaced this one. + if this.is_relinquished() { + return Err(this.relinquish_error()); + } + { *this.last_resume_request.lock().await = Timestamp::now_utc(); } @@ -1649,7 +1675,7 @@ impl Worker { OplogEntry::resumed(), None, ) - .await; + .await?; } let start_attempt = this.startup_attempt.begin(start_attempt); this.mark_as_loading(start_attempt); @@ -1959,7 +1985,23 @@ impl Worker { }; } + /// A start of an agent this executor has given up is never a success, and however it failed, + /// its waiters are told to retry on the shard's new owner. Without this a stop that reports a + /// generic "stopped before startup completed", or the recovery error a lost shard caused, + /// would hand them a failure they surface instead of retrying. + fn relinquished_startup_result( + &self, + result: Result<(), WorkerExecutorError>, + ) -> Result<(), WorkerExecutorError> { + if self.is_relinquished() { + Err(self.relinquish_error()) + } else { + result + } + } + fn publish_startup_result(&self, start_attempt: Uuid, result: Result<(), WorkerExecutorError>) { + let result = self.relinquished_startup_result(result); if !self.startup_attempt.complete(start_attempt, &result) { return; } @@ -1993,7 +2035,7 @@ impl Worker { _ => None, }; let is_active = active_attempt == Some(start_attempt); - let result = if is_active { + let mut result = if is_active { Ok(()) } else { Err(WorkerExecutorError::unknown( @@ -2012,13 +2054,21 @@ impl Worker { .last_error_kind == Some(OplogErrorKind::Recovery) { - self.add_and_commit_oplog_internal( - &instance_guard, - OplogEntry::recovery_succeeded(), - None, - ) - .await; + // Refused, the start is not a success: the agent has been given up, its waiters are + // told to look for the shard's new owner, and the caller stops it. + if self + .add_and_commit_oplog_internal( + &instance_guard, + OplogEntry::recovery_succeeded(), + None, + ) + .await + .is_err() + { + result = Err(WorkerExecutorError::ShardingNotReady); + } } + let result = self.relinquished_startup_result(result); let completed = match &result { Ok(()) => self @@ -2027,24 +2077,21 @@ impl Worker { Err(_) => self.startup_attempt.complete(start_attempt, &result), }; + let started = result.is_ok(); if completed { self.publish_completed_startup_result(start_attempt, result); } drop(instance_guard); - is_active + started } pub(crate) async fn record_recovery_failure(&self, error: &WorkerExecutorError) { // A recovery that failed because the shard moved is recorded by nobody: the agent is given - // up here and recovered by the shard's new owner. Marked rather than stopped, as the caller - // stops the worker right after this and that stop is where the agent is dropped. - if let Some(reason) = recovery_failure_relinquishment(error, self.oplog.fence()) { - self.mark_relinquished(reason); - return; - } - // Given up for a reason that never reached this error - a revoked or reassigned shard. The + // up here and recovered by the shard's new owner. The caller stops the worker right after + // this, and that stop is where the agent is dropped. An agent given up for a reason that + // never reached this error - a revoked or reassigned shard - is not recorded either: the // oplog would still accept the entry, at the epoch the agent no longer owns in spirit. - if self.is_relinquished() { + if self.relinquish_if_shard_lost(error) { return; } @@ -2068,15 +2115,17 @@ impl Worker { let error = recovery_agent_error(error); let retry_policy_state = (!infrastructure_failure && error != AgentError::OutOfMemory) .then_some(RetryPolicyState::Terminal); - self.add_and_commit_oplog(OplogEntry::error( - None, - OplogErrorKind::Recovery, - error, - retry_from, - false, - retry_policy_state, - )) - .await; + // A refusal marks the agent given up, which the caller's stop acts on. + let _ = self + .add_and_commit_oplog(OplogEntry::error( + None, + OplogErrorKind::Recovery, + error, + retry_from, + false, + retry_policy_state, + )) + .await; } pub(crate) fn pending_startup_attempt(&self) -> Option { @@ -2782,18 +2831,52 @@ impl Worker { /// /// The update itself is not performed by the invocation queue's processing loop, /// it is going to affect how the worker is recovered next time. - pub async fn enqueue_update(&self, update_description: UpdateDescription) { - // Bump + commit under the same instance lock. + pub async fn enqueue_update( + &self, + update_description: UpdateDescription, + ) -> Result<(), WorkerExecutorError> { let instance_guard = self.lock_non_stopping_worker().await; + self.enqueue_update_locked(&instance_guard, update_description) + .await + } + + /// Enqueues an update from inside this worker's own invocation loop. Returns `Ok(false)`, + /// enqueuing nothing, when the worker is stopping or has been given up, for the reasons given + /// on [`Self::cancel_invocation_from_loop`]. The manual update that produced the description + /// stays pending in the oplog and runs again in the next generation, here or on the shard's + /// new owner. + pub(crate) async fn enqueue_update_from_loop( + &self, + update_description: UpdateDescription, + ) -> Result { + let instance_guard = self.instance.lock().await; + if self.stopping_or_relinquished(&instance_guard) { + return Ok(false); + } + self.enqueue_update_locked(&instance_guard, update_description) + .await?; + Ok(true) + } + + fn stopping_or_relinquished(&self, instance_guard: &MutexGuard<'_, WorkerInstance>) -> bool { + matches!(&**instance_guard, WorkerInstance::Stopping(_)) || self.is_relinquished() + } + + async fn enqueue_update_locked( + &self, + instance_guard: &MutexGuard<'_, WorkerInstance>, + update_description: UpdateDescription, + ) -> Result<(), WorkerExecutorError> { + // Bump + commit under the same instance lock. self.bump_read_only_cache_epoch(); - let entry = OplogEntry::pending_update(update_description.clone()); + let entry = OplogEntry::pending_update(update_description); self.add_and_commit_oplog_internal( - &instance_guard, + instance_guard, entry, Some(WorkerCommand::WorkAvailable), ) - .await; - drop(instance_guard); + .await?; + Ok(()) } /// Enqueues a manual update. @@ -3471,7 +3554,7 @@ impl Worker { target_revision: ComponentRevision, new_component_size: u64, new_active_plugins: HashSet, - ) { + ) -> Result<(), OplogError> { let done = { let mut growth = self.memory_growth.lock().unwrap(); let entry = OplogEntry::successful_update( @@ -3484,12 +3567,13 @@ impl Worker { self.state_actor .queue_ordered_oplog_entry(self.clone(), entry) }; - if done.await.is_err() { + // A refusal is returned: an update the oplog does not record has not been applied. + done.await.unwrap_or_else(|_| { panic!( "Worker state actor for {} dropped an ordered oplog entry", self.owned_agent_id - ); - } + ) + }) } pub(crate) fn request_memory_limit_interrupt(self: &Arc, memory: LinearMemoryTracker) { @@ -3728,7 +3812,7 @@ impl Worker { entry, None, ) - .await; + .await?; } if let Some(idempotency_key) = semantic_idempotency_key { @@ -4249,7 +4333,7 @@ impl Worker { streams .commit_consumer_journal() .await - .map_err(WorkerExecutorError::runtime)?; + .map_err(|error| self.runtime_error_unless_fenced(error))?; } for mapping in &foreign_mappings { if producer.owns_handle_identity(&mapping.handle) @@ -4261,9 +4345,22 @@ impl Worker { continue; } let mut retry_delay = Duration::from_millis(10); + // Not bounded: the pending invocation and its attachment are already committed, so + // giving up would report a failure for an invocation the agent still runs. Only an + // agent this executor no longer owns stops retrying. loop { match streams.activate_foreign_mapping(mapping.clone(), 1).await { Ok(()) => break, + // A refusal is permanent. Retrying it would hold the instance lock forever, + // and the relinquish that has to take that lock could never stop the agent. + Err(_) if self.oplog.fence().is_some() => { + return Err(self.runtime_error_unless_fenced( + "durable foreign topology activation was fenced".to_string(), + )); + } + // A revoke writes nothing, so no fence latches: the mark is all there is, and + // `relinquish` is already waiting for this lock. + Err(_) if self.is_relinquished() => return Err(self.relinquish_error()), Err(error) => { warn!( session = %prepared.attempt.session_key.idempotency_key, @@ -4283,6 +4380,8 @@ impl Worker { .expect("persisted durable session has a commit notification") .send(()); } else if !attached_during_prepare { + // Refused, the invocation is not accepted: the dropped notification tells the waiter + // nothing was committed, and the error keeps `Accepted` from being sent. self.state_actor .commit_and_update_state_notifying( CommitLevel::Always, @@ -4290,7 +4389,8 @@ impl Worker { .take() .expect("legacy durable session has a commit notification"), ) - .await; + .await + .map_err(|fence| self.relinquished_by(fence))?; self.state_actor.notify_status_changed(); } if !already_attached && let WorkerInstance::Running(running) = &*instance_guard { @@ -4912,7 +5012,7 @@ impl Worker { let commit: DurableStreamCommit = Arc::new(move |committed| { let state_actor = state_actor.clone(); Box::pin(async move { - let (_, changed) = if let Some(committed) = committed { + let committed = if let Some(committed) = committed { state_actor .commit_and_update_state_notifying(CommitLevel::Always, committed) .await @@ -4921,7 +5021,10 @@ impl Worker { .commit_and_update_state(CommitLevel::Always) .await }; - if changed { + // A refusal is not reported through this closure: the producer reads it + // back from the oplog's latch, which the refused append set before the + // status actor replied, and the actor has spawned the relinquish. + if let Ok((_, true)) = committed { state_actor.notify_status_changed(); } }) @@ -5432,79 +5535,88 @@ impl Worker { } } - pub async fn commit_oplog_and_update_state(&self, commit_level: CommitLevel) -> OplogIndex { - let (result, changed) = self.state_actor.commit_and_update_state(commit_level).await; - if changed { - // The notification goes through the worker-state actor's lifecycle queue so that - // this method never waits on (or becomes a queued owner of) the instance lock. This - // method runs inside durable-call host futures polled by wasmtime's store event loop - // and on store-keeping wasm fibers, neither of which may block on locks shared with - // the other (see the `state_actor` module docs). - self.state_actor.notify_status_changed(); + /// Commits the buffered entries and folds them into the published status. + /// + /// A commit the storage refused is returned as `OplogError::Fenced`, and the agent is marked + /// relinquished. It has to be reported rather than folded into "nothing changed": below the + /// commit threshold an add only buffers, so this commit is where a takeover is found, and a + /// caller about to run a side effect, publish a result or acknowledge a request must not do + /// it for entries that never reached the storage. The status actor has already spawned the + /// relinquish that drops the agent from this executor. + pub async fn commit_oplog_and_update_state( + &self, + commit_level: CommitLevel, + ) -> Result { + match self.state_actor.commit_and_update_state(commit_level).await { + Ok((index, changed)) => { + if changed { + // The notification goes through the worker-state actor's lifecycle queue so + // that this method never waits on (or becomes a queued owner of) the instance + // lock. This method runs inside durable-call host futures polled by wasmtime's + // store event loop and on store-keeping wasm fibers, neither of which may block + // on locks shared with the other (see the `state_actor` module docs). + self.state_actor.notify_status_changed(); + } + Ok(index) + } + Err(fence) => Err(self.relinquished_by(fence)), } - result } - // Should only be called from invocation loop - pub async fn add_and_commit_oplog(&self, entry: OplogEntry) -> OplogIndex { - let result = self.add_to_oplog_or_relinquish(entry).await; + /// Adds an entry and commits it, reporting a refusal of either as `OplogError::Fenced` (see + /// [`Self::commit_oplog_and_update_state`]). + /// + /// Every other storage failure keeps the fail-stop behaviour of + /// [`Self::add_to_oplog_or_relinquish`]. + pub async fn add_and_commit_oplog(&self, entry: OplogEntry) -> Result { + let index = self.add_to_oplog_or_fenced(entry).await?; self.commit_oplog_and_update_state(CommitLevel::Always) - .await; - result + .await?; + Ok(index) } - /// Adds and commits an entry that gates a side effect - a durable scope `Start` or a remote - /// transaction's begin - and reports it as fenced when the storage refused it. - /// - /// `add_and_commit_oplog` cannot be used for these. Below the commit threshold an add only - /// buffers, so it answers with an index even on an oplog whose fence has latched, and the - /// status actor answers the refused commit that follows by spawning the relinquish rather - /// than failing. The caller would run its side effect for an entry that never reached the - /// storage, and the shard's new owner, finding no `Start`, would run it again. - /// - /// Every other storage failure keeps the fail-stop behaviour of `add_to_oplog_or_relinquish`. - pub async fn add_and_commit_oplog_or_fenced( - &self, - entry: OplogEntry, - ) -> Result { - let index = match self.oplog.add(entry).await { - Ok(index) => index, - Err(OplogError::Fenced(fence)) => { - self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence.clone())))); - return Err(OplogError::Fenced(fence)); - } + /// Appends an entry, reporting a refusal as `OplogError::Fenced` and marking the agent + /// relinquished; any other storage failure is fatal, as it always has been. + async fn add_to_oplog_or_fenced(&self, entry: OplogEntry) -> Result { + match self.oplog.add(entry).await { + Ok(index) => Ok(index), + Err(OplogError::Fenced(fence)) => Err(self.relinquished_by(fence)), Err(error) => panic!("oplog write: {error}"), - }; - self.commit_oplog_or_fenced(CommitLevel::Always).await?; - Ok(index) + } } - /// Commits the buffered entries, reporting a refused commit as fenced; for a commit a side - /// effect waits on (see [`Self::add_and_commit_oplog_or_fenced`]). - /// - /// The fence is read from the oplog's latch rather than from the commit, which swallows it. - /// That read is not early: the refused append latches the fence before the status actor - /// replies, and this awaits the reply. - pub async fn commit_oplog_or_fenced( - &self, - commit_level: CommitLevel, - ) -> Result { - let index = self.commit_oplog_and_update_state(commit_level).await; - let written = written_unless_fenced(index, self.oplog.fence()); - if let Err(OplogError::Fenced(fence)) = &written { - self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence.clone())))); + /// A failure that reached its caller flattened into a string, reported as the fence when the + /// oplog has latched one: the worker service retries a lost shard on its new owner, but not an + /// internal error. + fn runtime_error_unless_fenced(&self, error: String) -> WorkerExecutorError { + match self.oplog.fence() { + Some(fence) => self.relinquished_by(fence).into(), + None => WorkerExecutorError::runtime(error), } - written } - pub async fn queue_card_revocation(&self, card_id: CardId) -> Option { - self.queue_card_revocations(&[card_id]) - .await + /// Marks the agent given up because its oplog refused a write, and returns the refusal for the + /// caller to propagate. + pub(crate) fn relinquished_by(&self, fence: OplogFence) -> OplogError { + self.mark_relinquished(RelinquishReason::Fenced(Some(Box::new(fence.clone())))); + OplogError::Fenced(fence) + } + + pub async fn queue_card_revocation( + &self, + card_id: CardId, + ) -> Result, OplogError> { + Ok(self + .queue_card_revocations(&[card_id]) + .await? .into_iter() - .next() + .next()) } - pub async fn queue_card_revocations(&self, card_ids: &[CardId]) -> Vec { + pub async fn queue_card_revocations( + &self, + card_ids: &[CardId], + ) -> Result, OplogError> { let boundary_lock = self.card_event_boundary_lock.clone(); let _boundary_guard = boundary_lock.lock().await; self.queue_card_revocations_locked(card_ids).await @@ -5513,7 +5625,7 @@ impl Worker { pub(crate) async fn queue_card_revocations_locked( &self, card_ids: &[CardId], - ) -> Vec { + ) -> Result, OplogError> { let status = self.get_last_known_status().await; let pending_revocations = status .pending_card_events @@ -5538,19 +5650,19 @@ impl Worker { let mut queued_event_indices = Vec::with_capacity(card_ids.len()); for card_id in card_ids { queued_event_indices.push( - self.add_to_oplog_or_relinquish(OplogEntry::card_event_queued( + self.add_to_oplog_or_fenced(OplogEntry::card_event_queued( None, QueuedCardEvent::revoke(card_id), )) - .await, + .await?, ); } if !queued_event_indices.is_empty() { self.commit_oplog_and_update_state(CommitLevel::Always) - .await; + .await?; } - queued_event_indices + Ok(queued_event_indices) } pub async fn receive_card_transfer( @@ -5624,13 +5736,18 @@ impl Worker { self.published_authority_generation.clone() } + /// [`Self::add_and_commit_oplog`] for a caller holding the instance lock, which sends + /// `wakeup` to the running loop when the commit changed the status. + /// + /// A refusal is returned rather than acknowledged: the management request that wrote the + /// entry must not report it as accepted. async fn add_and_commit_oplog_internal( &self, instance_guard: &MutexGuard<'_, WorkerInstance>, entry: OplogEntry, wakeup: Option, - ) -> OplogIndex { - let result = self.add_to_oplog_or_relinquish(entry).await; + ) -> Result { + let index = self.add_to_oplog_or_fenced(entry).await?; // The caller already holds the instance lock (and sends the wakeup itself below), so // this must not enqueue a `NotifyStatusChanged` lifecycle job: the commit job is safe to // await while holding the instance lock precisely because the status task never takes @@ -5638,7 +5755,8 @@ impl Worker { let (_, changed) = self .state_actor .commit_and_update_state(CommitLevel::Always) - .await; + .await + .map_err(|fence| self.relinquished_by(fence))?; if changed && let Some(wakeup) = wakeup @@ -5647,7 +5765,7 @@ impl Worker { running.sender.send(wakeup).unwrap(); }; - result + Ok(index) } async fn activate_plugin_internal( @@ -5669,7 +5787,7 @@ impl Worker { OplogEntry::activate_plugin(plugin_grant_id), Some(WorkerCommand::WorkAvailable), ) - .await; + .await?; drop(instance_guard); Ok(()) @@ -5694,7 +5812,7 @@ impl Worker { OplogEntry::deactivate_plugin(plugin_grant_id), Some(WorkerCommand::WorkAvailable), ) - .await; + .await?; drop(instance_guard); Ok(()) @@ -5740,7 +5858,36 @@ impl Worker { idempotency_key: IdempotencyKey, ) -> Result<(), WorkerExecutorError> { let instance_guard = self.lock_non_stopping_worker().await; + self.cancel_invocation_locked(&instance_guard, idempotency_key) + .await + } + + /// Cancels a pending invocation from inside this worker's own invocation loop. Returns + /// `Ok(false)`, cancelling nothing, when the worker is stopping or has been given up. + /// + /// [`Self::cancel_invocation`] waits for a stop to finish, and a stop from outside the loop + /// finishes only once the loop has exited, so the loop waiting for it would never return. The + /// loop exits instead; the invocation stays pending in the oplog and the next generation + /// skips it again. A given-up agent's oplog is the new owner's to write, so nothing is + /// written for it even before its stop begins. + pub(crate) async fn cancel_invocation_from_loop( + &self, + idempotency_key: IdempotencyKey, + ) -> Result { + let instance_guard = self.instance.lock().await; + if self.stopping_or_relinquished(&instance_guard) { + return Ok(false); + } + self.cancel_invocation_locked(&instance_guard, idempotency_key) + .await?; + Ok(true) + } + async fn cancel_invocation_locked( + &self, + instance_guard: &MutexGuard<'_, WorkerInstance>, + idempotency_key: IdempotencyKey, + ) -> Result<(), WorkerExecutorError> { if instance_guard.is_deleting() { return Err(WorkerExecutorError::invalid_request( "Cannot cancel invocation on a deleting worker", @@ -5748,13 +5895,11 @@ impl Worker { }; self.add_and_commit_oplog_internal( - &instance_guard, + instance_guard, OplogEntry::cancel_pending_invocation(idempotency_key), Some(WorkerCommand::WorkAvailable), ) - .await; - - drop(instance_guard); + .await?; Ok(()) } @@ -5896,7 +6041,7 @@ impl Worker { OplogEntry::revert(dropped_region), None, ) - .await; + .await?; self.reattach_worker_status().await; self.current_component.store(Arc::new(restored_component)); @@ -6027,6 +6172,12 @@ impl Worker { loop { match self.lookup_invocation_result(key).await { LookupResult::Interrupted => break Ok(LookupResult::Interrupted), + // Given up here, so no result for the key will be published on this executor. The + // retry answer is published when the agent is given up but not cached, and a + // receiver that lagged past it, or subscribed after it, finds it here instead. + LookupResult::New | LookupResult::Pending if self.is_relinquished() => { + break Ok(LookupResult::Complete(Err(self.relinquish_error()))); + } LookupResult::New | LookupResult::Pending => { let waiting = subscription.wait_for(|event| match event { Event::InvocationCompleted { @@ -6059,6 +6210,13 @@ impl Worker { next_ownership_check = tokio::time::Instant::now() + INVOCATION_OWNERSHIP_RECHECK_INTERVAL; + // A key the give-up did not know about yet (enqueued, not yet folded + // into the status it failed from) gets no retry answer published. + // The lookup at the top of the loop answers it. + if self.is_relinquished() { + continue; + } + // An agent whose shard has moved is resumed by whoever owns // it now, and its `InvocationCompleted` is published on that // executor's bus. Nothing will ever arrive on ours, and the @@ -6436,7 +6594,7 @@ impl Worker { true } Err(error) => { - warn!("Committing the oplog while stopping failed: {error}"); + warn!(%error, "Committing the oplog while stopping failed"); false } }; @@ -6623,6 +6781,19 @@ impl Worker { } async fn fail_pending_invocations(&self, error: WorkerExecutorError) { + // A relinquished agent's pending invocations are not failed, they move: the shard's new + // owner runs them. So their waiters are told to retry there, whatever stopped this + // generation, and no result is cached. A cached failure would outlive the stop: a later + // lookup would answer the key with an `InvocationFailed` the caller does not retry, and + // the invocation loop, finding the key complete, would cancel the pending invocation - + // waiting on this very stop to do it, or, with nothing fenced yet, cancelling work the new + // owner still has to run. + let relinquished = self.is_relinquished(); + let error = if relinquished { + self.relinquish_error() + } else { + error + }; let queued_items = self.queue.write().await.drain(..).collect::>(); let mut origins = self.external_invocation_origins.write().await; @@ -6656,6 +6827,11 @@ impl Worker { { continue; } + if relinquished { + self.publish_completion(idempotency_key, Err(error.clone())); + origins.remove(idempotency_key); + continue; + } invocation_results.insert( idempotency_key.clone(), InvocationResult::Cached { @@ -7420,19 +7596,6 @@ struct PendingWorkerInterrupt { unload_request: UnloadRequest, } -/// Whether an entry the oplog answered with `index` was written, given the fence the oplog has -/// latched by the time its commit returned. A latched fence means the commit was refused, whatever -/// the add answered, so the index is not handed to a caller about to run a side effect. -fn written_unless_fenced( - index: OplogIndex, - latched: Option, -) -> Result { - match latched { - Some(fence) => Err(OplogError::Fenced(fence)), - None => Ok(index), - } -} - /// The shard epoch the agent's oplog is opened to assert, read from this executor's current /// assignment. See [`shard_epoch_to_assert`]. fn owned_shard_epoch( @@ -7935,12 +8098,15 @@ impl RunningWorker { "Attempting update to revision {component_revision} failed with {error}" ); + // Refused, the update cannot be marked failed, and retrying would find + // the same pending update again: the start fails as a lost shard instead. parent .add_and_commit_oplog(OplogEntry::failed_update( component_revision, Some(error.to_string()), )) - .await; + .await + .map_err(WorkerExecutorError::from)?; // The update is now marked failed in the parent, we can retry. return Box::pin(Self::create_instance(parent, concurrent_agent_permit)) @@ -8962,27 +9128,6 @@ mod tests { use std::path::Path; use test_r::test; - /// The add that buffered a side effect's entry answered with an index. A fence latched by the - /// commit must win over that answer, or the side effect runs for an entry nobody can see. - #[test] - fn a_latched_fence_refuses_an_entry_its_add_accepted() { - let fence = OplogFence { - agent_id: AgentId { - component_id: ComponentId(Uuid::new_v4()), - agent_id: "gated".to_string(), - }, - expected_epoch: ShardEpoch(8), - actual_epoch: Some(ShardEpoch(9)), - }; - let index = OplogIndex::from_u64(5); - - assert_eq!(written_unless_fenced(index, None), Ok(index)); - assert_eq!( - written_unless_fenced(index, Some(fence.clone())), - Err(OplogError::Fenced(fence)) - ); - } - /// Admission and the epoch read are not atomic. An agent whose shard left the assignment in /// between must be refused, not handed an oplog that asserts nothing. #[test] @@ -9123,13 +9268,15 @@ mod tests { )); } - /// A recovery failure is persisted as an `Error` entry, which a lost shard makes unwritable: - /// the entry belongs to an oplog whose owner has changed. + /// A lost shard is neither recorded nor retried in place, whichever path it failed: a recovery + /// `Error` entry, like an in-place retry of a streaming-session completion, would write to or + /// reopen an oplog whose owner has changed. Every path classifies with the same predicate, so + /// the shapes one of them recognises are recognised by all. #[test] - fn a_recovery_failure_that_is_a_lost_shard_is_given_up_rather_than_persisted() { + fn a_failure_that_is_a_lost_shard_is_given_up_on_every_path() { let agent_id = AgentId { component_id: ComponentId::new(), - agent_id: "recovering".to_string(), + agent_id: "fenced".to_string(), }; let fence = OplogFence { agent_id: agent_id.clone(), @@ -9137,23 +9284,26 @@ mod tests { actual_epoch: Some(ShardEpoch(3)), }; - // A latched fence wins over whatever the refusal was flattened into on its way out. + // A latched fence wins over whatever the refusal was flattened into on its way out, and + // its details are kept for the error the waiters are given. assert!(matches!( - recovery_failure_relinquishment( - &WorkerExecutorError::runtime("flattened"), - Some(fence) + shard_lost_relinquishment( + &WorkerExecutorError::runtime("durable stream commit failed"), + Some(fence.clone()) ), - Some(RelinquishReason::Fenced(Some(_))) + Some(RelinquishReason::Fenced(Some(latched))) if *latched == fence )); assert!(matches!( - recovery_failure_relinquishment( + shard_lost_relinquishment( &WorkerExecutorError::oplog_fenced(agent_id, 2, Some(3)), None ), Some(RelinquishReason::Fenced(None)) )); + // Without a latched fence: a shard revoked or reassigned interrupts the agent with + // `ShardLost` and writes nothing, so no fence ever latches. assert!(matches!( - recovery_failure_relinquishment( + shard_lost_relinquishment( &WorkerExecutorError::Interrupted { kind: InterruptKind::ShardLost }, @@ -9162,10 +9312,20 @@ mod tests { Some(RelinquishReason::Fenced(None)) )); - // Every other recovery failure is still the agent's own, and is persisted. - assert!( - recovery_failure_relinquishment(&WorkerExecutorError::runtime("boom"), None).is_none() - ); + // Every other failure is still the agent's own, to be recorded or retried. + for kind in [ + InterruptKind::Interrupt(Timestamp::now_utc()), + InterruptKind::Suspend(Timestamp::now_utc()), + InterruptKind::Restart, + InterruptKind::Jump, + ] { + assert!( + shard_lost_relinquishment(&WorkerExecutorError::Interrupted { kind }, None) + .is_none(), + "{kind:?} is not a lost shard" + ); + } + assert!(shard_lost_relinquishment(&WorkerExecutorError::runtime("boom"), None).is_none()); } #[test] diff --git a/golem-worker-executor/src/worker/state_actor.rs b/golem-worker-executor/src/worker/state_actor.rs index 0a20974358..8418bbf99b 100644 --- a/golem-worker-executor/src/worker/state_actor.rs +++ b/golem-worker-executor/src/worker/state_actor.rs @@ -105,13 +105,14 @@ pub(crate) struct OwnerCommitController { /// unpollable callers. enum StatusJob { /// Commits the oplog and folds the newly committed entries into the published status. - /// Replies with the current oplog index after the commit and whether the status changed. + /// Replies with the current oplog index after the commit and whether the status changed, or + /// with the fence when the storage refused the commit. /// The reply deliberately does not depend on the instance lock; if the caller wants the /// invocation loop notified about the change, it enqueues a lifecycle job afterwards. CommitAndUpdateState { level: CommitLevel, committed: Option>, - done: oneshot::Sender<(OplogIndex, bool)>, + done: oneshot::Sender>, }, /// Appends an entry and completes its commit + fold transaction even if the caller is /// cancelled. The caller-acquired guards remain owned by this job until the transaction ends. @@ -164,7 +165,7 @@ enum LifecycleJob { OrderedOplogEntry { worker: Arc>, entry: Box, - done: oneshot::Sender<()>, + done: oneshot::Sender>, }, MemoryLimitExceeded { worker: Arc>, @@ -243,14 +244,10 @@ impl WorkerStateActor { } => { complete_status_job( async { - // The reply keeps its shape: these callers learn of a fence from - // the oplog's latch and from the relinquish the refusal spawned. - let changed = state - .commit_and_update_state(level, committed) - .await - .unwrap_or(false); + let changed = + state.commit_and_update_state(level, committed).await?; let index = state.oplog.current_oplog_index().await; - (index, changed) + Ok((index, changed)) }, done, ) @@ -380,8 +377,7 @@ impl WorkerStateActor { entry, done, } => { - worker.add_and_commit_oplog(*entry).await; - let _ = done.send(()); + let _ = done.send(worker.add_and_commit_oplog(*entry).await.map(|_| ())); } LifecycleJob::MemoryLimitExceeded { worker, memory } => { worker @@ -414,11 +410,15 @@ impl WorkerStateActor { } /// Commits the oplog and folds the new entries into the published status. Returns the - /// current oplog index after the commit and whether the status changed. + /// current oplog index after the commit and whether the status changed, or the fence when the + /// storage refused the commit; the refusal has already spawned the agent's relinquish. /// /// If the caller's future is dropped while awaiting the reply, the commit still runs to /// completion on the status task (the same semantics as the oplog actor's own jobs). - pub async fn commit_and_update_state(&self, level: CommitLevel) -> (OplogIndex, bool) { + pub async fn commit_and_update_state( + &self, + level: CommitLevel, + ) -> Result<(OplogIndex, bool), OplogFence> { self.commit .run_status_job(|done| StatusJob::CommitAndUpdateState { level, @@ -432,7 +432,7 @@ impl WorkerStateActor { &self, level: CommitLevel, committed: oneshot::Sender<()>, - ) -> (OplogIndex, bool) { + ) -> Result<(OplogIndex, bool), OplogFence> { self.commit .run_status_job(|done| StatusJob::CommitAndUpdateState { level, @@ -556,7 +556,7 @@ impl WorkerStateActor { &self, worker: Arc>, entry: OplogEntry, - ) -> oneshot::Receiver<()> { + ) -> oneshot::Receiver> { let (done, done_rx) = oneshot::channel(); if self .lifecycle_jobs @@ -577,7 +577,10 @@ impl WorkerStateActor { } impl OwnerCommitController { - pub async fn commit_and_update_state(&self, level: CommitLevel) -> (OplogIndex, bool) { + pub async fn commit_and_update_state( + &self, + level: CommitLevel, + ) -> Result<(OplogIndex, bool), OplogFence> { self.run_status_job(|done| StatusJob::CommitAndUpdateState { level, committed: None, diff --git a/golem-worker-executor/src/workerctx/default.rs b/golem-worker-executor/src/workerctx/default.rs index 70e7792188..3421f2394f 100644 --- a/golem-worker-executor/src/workerctx/default.rs +++ b/golem-worker-executor/src/workerctx/default.rs @@ -658,7 +658,7 @@ impl UpdateManagement for Context { &self, target_revision: ComponentRevision, details: Option, - ) { + ) -> Result<(), WorkerExecutorError> { self.durable_ctx .on_worker_update_failed(target_revision, details) .await @@ -669,7 +669,7 @@ impl UpdateManagement for Context { target_revision: ComponentRevision, new_component_size: u64, new_active_plugins: HashSet, - ) { + ) -> Result<(), WorkerExecutorError> { self.durable_ctx .on_worker_update_succeeded(target_revision, new_component_size, new_active_plugins) .await diff --git a/golem-worker-executor/src/workerctx/mod.rs b/golem-worker-executor/src/workerctx/mod.rs index 5fb4b51f99..162bb2380f 100644 --- a/golem-worker-executor/src/workerctx/mod.rs +++ b/golem-worker-executor/src/workerctx/mod.rs @@ -518,20 +518,22 @@ pub trait UpdateManagement { /// Marks the end of a snapshot function call. This can be used to re-enable persistence fn end_call_snapshotting_function(&mut self); - /// Called when an update attempt has failed + /// Called when an update attempt has failed. Fails when the oplog refused to record the + /// failure: the agent has been given up, and must not be rebuilt on its old revision here. async fn on_worker_update_failed( &self, target_revision: ComponentRevision, details: Option, - ); + ) -> Result<(), WorkerExecutorError>; - /// Called when an update attempt succeeded + /// Called when an update attempt succeeded. Fails when the oplog refused to record the + /// update: the agent has been given up, and the update must not be reported as applied. async fn on_worker_update_succeeded( &self, target_revision: ComponentRevision, new_component_size: u64, new_active_plugins: HashSet, - ); + ) -> Result<(), WorkerExecutorError>; } /// Operations not requiring an active worker context, but still depending on the diff --git a/golem-worker-executor/tests/api.rs b/golem-worker-executor/tests/api.rs index ef3f68de53..88561db943 100644 --- a/golem-worker-executor/tests/api.rs +++ b/golem-worker-executor/tests/api.rs @@ -5924,6 +5924,42 @@ async fn a_caller_is_answered_when_its_agents_shard_comes_back( elements: vec![SchemaValue::U8(42)] } ); + + // The value alone cannot tell the retry being handed the parked call's own answer apart from + // a second, independent run of `await_promise` that happened to read the same completed + // promise: both return `[42]`. Count the method's oplog pair instead - the idempotency key + // must have deduplicated the retry into the original invocation, so there can only be one. + use golem_common::model::oplog::{PublicAgentInvocation, PublicOplogEntry}; + let oplog = executor + .get_oplog(&parked.agent_id, OplogIndex::INITIAL) + .await?; + let started = oplog + .iter() + .filter(|entry| match &entry.entry { + PublicOplogEntry::AgentInvocationStarted(params) => matches!( + ¶ms.invocation, + PublicAgentInvocation::AgentMethodInvocation(m) + if m.method_name.replace('-', "_") == "await_promise" + ), + _ => false, + }) + .count(); + let finished = oplog + .iter() + .filter(|entry| match &entry.entry { + PublicOplogEntry::AgentInvocationFinished(params) => params + .method_name + .as_deref() + .is_some_and(|name| name.replace('-', "_") == "await_promise"), + _ => false, + }) + .count(); + assert_eq!( + (started, finished), + (1, 1), + "the retry under the same idempotency key must be answered from the recorded run, not \ + by executing await_promise a second time" + ); Ok(()) } @@ -6391,6 +6427,9 @@ async fn a_caller_waiting_on_an_invocation_fenced_inside_a_host_call_is_told_to_ // Lets `subscribe_duration`'s own commit finish, so the guest is parked in `poll` with about // five seconds of sleep left and nothing else is due to write. sleep(Duration::from_secs(1)).await; + // Baseline for the no-further-progress check below: from here on, the guest's only next + // durable operation is the monotonic-clock read that must be refused. + let oplog_before_takeover = executor.get_oplog(&worker_id, OplogIndex::INITIAL).await?; let mut clock = executor .gate_next_monotonic_clock_start(&owned_agent_id) @@ -6425,9 +6464,20 @@ async fn a_caller_waiting_on_an_invocation_fenced_inside_a_host_call_is_told_to_ .await .map_err(|_| anyhow!("the fenced agent stayed cached on this executor"))?; - // The hook signals `entered` only once its commit has succeeded, and by now the agent is gone - // from this executor, so a gate that was going to be entered already has been. Silence means - // the refusal happened at that commit, inside the host call. + // `entered()` only fires once the gated commit *succeeds* (`OwnerExecution:: + // test_after_monotonic_clock_start` sends it after the commit, not before), so its timing out + // here does not by itself prove the guest ever reached the gate: the same timeout would be + // observed if the agent had been interrupted by something else first, well before the + // monotonic-clock read. Prove reachability directly from the oplog instead: nothing may follow + // `subscribe_duration`'s `End` on this executor once the shard is taken away, since the + // guest's only next durable operation is the monotonic-clock read the fence must have refused. + let oplog_after = executor.get_oplog(&worker_id, OplogIndex::INITIAL).await?; + assert_eq!( + oplog_after.len(), + oplog_before_takeover.len(), + "no oplog entries may follow `subscribe_duration`'s `End` once the shard is taken away; \ + the guest's next durable call is the monotonic-clock read the fence must have refused" + ); assert!( tokio::time::timeout(Duration::from_millis(500), clock.entered()) .await diff --git a/golem-worker-executor/tests/instance_layer.rs b/golem-worker-executor/tests/instance_layer.rs index d58cc73fa6..9ee0788e42 100644 --- a/golem-worker-executor/tests/instance_layer.rs +++ b/golem-worker-executor/tests/instance_layer.rs @@ -592,7 +592,11 @@ async fn completed_tool_positional_replay_errors( )?; live.await_result(&parent_id).await?; drop(primary); - active_agent.execution().commit(CommitLevel::Always).await; + active_agent + .execution() + .commit(CommitLevel::Always) + .await + .unwrap(); let before = active_agent.execution().oplog().current_oplog_index().await; active_agent @@ -781,7 +785,11 @@ async fn incomplete_tool_config_tail_reauthorizes_without_rejecting_recorded_rep )?; live.await_result(&parent_id).await?; drop(primary); - active_agent.execution().commit(CommitLevel::Always).await; + active_agent + .execution() + .commit(CommitLevel::Always) + .await + .unwrap(); active_agent .execution() @@ -2088,7 +2096,11 @@ async fn filesystem_capable_entity_stream_replays_on_owner_filesystem( )?; let live_result = live.await_result(&parent_id).await?; drop(primary); - active_agent.execution().commit(CommitLevel::Always).await; + active_agent + .execution() + .commit(CommitLevel::Always) + .await + .unwrap(); active_agent .execution() diff --git a/golem-worker-executor/worker-executor-walkthrough.html b/golem-worker-executor/worker-executor-walkthrough.html index 8d1b58edfd..9b7e179264 100644 --- a/golem-worker-executor/worker-executor-walkthrough.html +++ b/golem-worker-executor/worker-executor-walkthrough.html @@ -374,7 +374,7 @@

Replay is deterministic by construction

2

The resident runtime is disposable

-

The Wasmtime Store, the worker task, sockets, channels, caches and the executor process itself may vanish at any instruction boundary. Suspend, evict, reshard, restart and crash are all recovered the same way: throw the instance away, build a new Store, replay the oplog, continue.

+

The Wasmtime Store, the worker task, sockets, channels, caches and the executor process itself may vanish at any instruction boundary. Suspend, evict, restart and crash are all recovered the same way, by the same executor: throw the instance away, build a new Store, replay the oplog, continue. Losing the shard is different: every oplog write asserts this executor's shard epoch inside the storage transaction, and once another executor holds it, the next write is refused rather than accepted. That refusal relinquishes the agent (InterruptKind::ShardLost) — stopped here with nothing more written, dropped from this executor, never restarted in place — and it is the shard's new owner, not this executor, that builds the Store and replays (§18).

3
@@ -455,7 +455,7 @@

2Architecture & component m
Golem services around the worker executor - Clients call the worker service over HTTP or gRPC. The worker service asks the shard manager which executor owns an agent's shard and forwards the invocation to that executor over gRPC. Inside the executor, a Worker object per agent owns an invocation queue, a Wasmtime Store and a durable host context; the durable host context appends to and replays from the oplog service, which is backed by indexed storage (Redis or SQLite) with archive layers in blob storage. Executors also talk to the registry service for components and metadata, and to each other for RPC. + Clients call the worker service over HTTP or gRPC. The worker service asks the shard manager which executor owns an agent's shard and forwards the invocation to that executor over gRPC. Inside the executor, a Worker object per agent owns an invocation queue, a Wasmtime Store and a durable host context; the durable host context appends to and replays from the oplog service, which is backed by indexed storage (Postgres or SQLite alongside a real shard manager, since only those can fence a write on the shard epoch; Redis cannot and is refused at startup in that topology) with archive layers in blob storage. Executors also talk to the registry service for components and metadata, and to each other for RPC. ClientsCLI, HTTP API, SDKs Worker servicerouting, API gateway @@ -477,7 +477,7 @@

2Architecture & component m Durable host context (DurableWorkerCtx)durability guard · durable call sessions · replay cursorhost APIs: http, key-value, rpc, tools, streams… Servicesscheduler, promises, rpc,events, plugins - Oplog service (multi-layer)primary: indexed storage (Redis or SQLite)archive layers: compressed, then blob storage + Oplog service (multi-layer)primary: indexed storage (Postgres or SQLite; fenced on shard epoch)archive layers: compressed, then blob storage Other executorsdurable RPCstream transportworker proxy @@ -574,7 +574,7 @@

Cross-agent plumbing services/{rpc,promise,scheduler,worke

Storage backends src/storage/{indexed,keyvalue,scheduler}/*

-

The oplog service, key-value service, scheduler and status flusher are written against three small storage traits. Implementations: in-memory (tests), SQLite and multi-SQLite (single binary, local runs), PostgreSQL and Redis (clusters). Decorators exist for retrying, namespace routing and fault injection (used by the tests that simulate storage failure). Blob storage for oplog archive layers and snapshots is provided by golem-service-base.

+

The oplog service, key-value service, scheduler and status flusher are written against three small storage traits. Implementations: in-memory (tests), SQLite and multi-SQLite (single binary, local runs), PostgreSQL and Redis. A real shard manager moves shards between executors, so its oplog writes must be fenceable on the shard epoch (IndexedStorage::supports_epoch_fencing) or the executor refuses to start; only PostgreSQL and the SQLite-backed indexed storages qualify — Redis remains available for key-value storage and for indexed storage without a real shard manager (single-shard, debugging service), but not for a clustered executor's oplog. Decorators exist for retrying, namespace routing and fault injection (used by the tests that simulate storage failure). Blob storage for oplog archive layers and snapshots is provided by golem-service-base.

@@ -742,6 +742,8 @@

Append versus commit

CommitLevel::Always waits for durable storage; CommitLevel::DurableOnly only does so for durable agents (the ephemeral oplog honours the level, the primary oplog always flushes). Whenever this document says "accepted only after commit", it means the commit, not the append.

+

A commit can be refused instead of written. Every append and commit asserts this executor's shard epoch inside the storage transaction, and a storage that has recorded a newer epoch — another executor now owns the shard — returns OplogError::Fenced (DurableStreamProducerError::Fenced for durable-stream session records) instead of writing anything. commit_oplog_and_update_state and add_and_commit_oplog return that refusal rather than swallowing it, so the three guarantees above hold either way: a refused PendingAgentInvocation commit is not acknowledged as accepted, a refused AgentInvocationFinished commit is not published to waiters, and a refused side-effect Start is not treated as durable. The first refusal latches — every later add on that oplog is refused too, without a second storage round trip — and the agent relinquishes (§18) instead of retrying the write or restarting in place.

+

worker/state_actor.rs::commit_and_update_state samples the appended oplog tip before its explicit commit and discards receipt entries already folded into the published status. Threshold auto-flushes in both primary and ephemeral oplogs, and replica waits, can commit outside the status actor; therefore an empty commit receipt does not prove that no new entries committed. If the remaining receipt is not exactly the contiguous suffix after the last published status, the status actor reuses status::try_fold_status_from to read committed storage in bounded chunks, hydrate external StreamSession payloads and publish the folded status once. This prevents unbounded accumulation of auto-flushed entries. Gap recovery conservatively invalidates authority snapshots after the fold. Ephemeral DurableOnly deliberately remains non-flushing and preserves its no-I/O fast path. The recovery adds no oplog entry or protocol change and is status reconstruction, not replay tolerance.

Rediscovery

@@ -1819,7 +1821,8 @@

18Interrupt, suspend, restart, evic SuspendInterruptKind::Suspend(ts), used when the guest sleeps or waits for a promise long enough to unloadSuspend (h)On demand: a new invocation, a promise completion, or a ScheduledAction::Resume JumpInterruptKind::Jump ("jumping back in time"): atomic-region rollback or set-oplog-indexJump { region } (positional)Immediately, replaying with the region skipped Eviction (memory / filesystem pressure)EvictionClass ordering: LoadedIdle first (cheapest), then WarmRunnable (has durable pending invocations). Executing workers and workers with non-durable in-memory work (internal queue, ResumeReplay, interrupt) are never evicted.noneOn the next invocation or scheduled action - Reshardingon_shard_assignment_changed: workers whose shard moved away are unloaded here and reconstructed by the new ownernoneWhen the new owner is asked for the worker + ReshardingThe shard manager's RevokeShards/AssignShards gRPC calls relinquish (RelinquishReason::ShardRevoked / ShardNotAssigned) every agent whose shard moved away, via InterruptKind::ShardLostnoneNever on this executor. The agent is stopped and dropped here; the new owner is the one that reconstructs, when the worker service sends it a request + Oplog epoch fenceA write is refused because the shard epoch this executor asserted no longer matches storage (OplogError::Fenced / OplogFence); same relinquish path as resharding, via RelinquishReason::Fencednone — the fence latches, so nothing later is written eitherNever on this executor, for the same reason. Any invocation still pending here is failed with a retriable error (no cached result), so worker-service retries it against the new owner instead Process death—whatever was committedWhen any executor is asked for the worker diff --git a/local-run/start.sh b/local-run/start.sh index 4832826a51..3d02a9fb40 100644 --- a/local-run/start.sh +++ b/local-run/start.sh @@ -184,6 +184,10 @@ GOLEM__BLOB_STORAGE__CONFIG__ROOT="${FS_BLOB_STORAGE_DIR}" \ GOLEM__REGISTRY_SERVICE__HOST="localhost" \ GOLEM__REGISTRY_SERVICE__PORT=${REGISTRY_SERVICE_GRPC_PORT} \ GOLEM__CORS_ORIGIN_REGEX="http://localhost:3000" \ +GOLEM__INDEXED_STORAGE__TYPE="Sqlite" \ +GOLEM__INDEXED_STORAGE__CONFIG__DATABASE="../local-run/data/worker-executor/golem_indexed.sqlite" \ +GOLEM__INDEXED_STORAGE__CONFIG__MAX_CONNECTIONS=10 \ +GOLEM__INDEXED_STORAGE__CONFIG__FOREIGN_KEYS=false \ ../target/debug/golem-debugging-service & debugging_service_pid=$! From 9e7642854981c526a6b0833a42570a7c56988273 Mon Sep 17 00:00:00 2001 From: Aditya Salunkhe Date: Fri, 18 Sep 2026 15:33:22 +0530 Subject: [PATCH 5/6] Fixes for findings --- docs/src/content/next/deploy.mdx | 6 +- .../src/model/invocation_session_public.rs | 7 + golem-common/src/model/quota.rs | 13 +- .../src/quota/quota_service.rs | 2 +- golem-shard-manager/src/quota/quota_state.rs | 18 +- golem-shard-manager/src/sharding/model.rs | 202 +++++++++--- .../src/sharding/shard_management.rs | 2 +- .../indexed/postgres/002_oplog_metadata.sql | 10 +- .../indexed/sqlite/002_oplog_metadata.sql | 4 +- .../src/durable_host/concurrent/tests.rs | 1 + .../src/durable_host/durable_session/tests.rs | 143 ++++++++- .../src/durable_host/durable_stream/tests.rs | 9 +- golem-worker-executor/src/durable_host/mod.rs | 19 ++ .../src/durable_host/p3/http/send.rs | 1 + golem-worker-executor/src/model/mod.rs | 2 + .../src/services/oplog/mod.rs | 4 + .../src/services/oplog/plugin.rs | 1 + .../src/services/oplog/primary.rs | 13 +- .../src/services/oplog/tests.rs | 4 + golem-worker-executor/src/services/shard.rs | 13 +- .../src/services/shard_manager.rs | 1 + .../src/storage/indexed/mod.rs | 74 ++++- .../src/storage/indexed/multi_sqlite.rs | 23 +- .../src/storage/indexed/postgres.rs | 93 ++++-- .../src/storage/indexed/sqlite.rs | 88 +++-- golem-worker-executor/src/worker/mod.rs | 2 + golem-worker-executor/tests/api.rs | 168 +++++++++- golem-worker-executor/tests/hot_update.rs | 101 ++++++ .../tests/indexed_storage.rs | 302 +++++++++++++++++- .../src/api/invocation_session.rs | 6 + 30 files changed, 1206 insertions(+), 126 deletions(-) diff --git a/docs/src/content/next/deploy.mdx b/docs/src/content/next/deploy.mdx index 6c6e2b6af0..f377db2c8d 100644 --- a/docs/src/content/next/deploy.mdx +++ b/docs/src/content/next/deploy.mdx @@ -57,11 +57,11 @@ See also: [configuration](https://github.com/golemcloud/golem/blob/main/golem-wo Every worker executor holds a **lease** on its shard assignment and renews it with the shard manager. A lease lasts `shard_lease_duration` (1m by default) and an executor renews at a third of the time it has left, so roughly every 20s. The shard manager refuses to start with a lease shorter than 30s and warns about one shorter than 1m: the deadline it allows each renewal call never drops below 10s, so a shorter lease cannot fit the three renewal attempts a lease is meant to get. Each renewal carries the epoch of every shard the executor holds; a shard's epoch is its ownership generation, advanced when the shard moves to another executor and otherwise only to repair a shard manager state that lost history, as described under oplog fencing below. A renewal whose claim does not match the shard manager's view is renewed all the same, and the response carries the shard manager's set, which the executor adopts exactly as it would an assignment push: pushes deliver a change at once, and renewals guarantee it arrives within a third of the lease even if the push was lost. The shard manager records a shard's new owner before it tells anybody about it, so a rebalance whose write is refused changes nothing at all, and every delivery carries the revision of the state it was read from. An executor ignores a delivery older than one it has already applied, so a push and a renewal that cross on the network cannot leave the older set in place. An executor that stops renewing loses its lease, and the shard manager reclaims and redistributes those shards on its next pass, which runs at least every third of `shard_lease_duration` even in a cluster where nothing else is happening; a crashed or unreachable executor's shards are therefore re-homed within about one lease duration plus one pass. The lease reaches the executor as the time remaining on it rather than as an absolute time, so the two machines' clocks are never compared and skew between them can neither lengthen nor shorten it. The executor anchors that time to the moment it sent the request the lease answers, so time the reply spends in flight comes off its copy of the lease and is never added to it, and its copy lapses no later than the shard manager's. An assignment push carries the shard set and its revision only: it never extends a lease. The lease is the one place the two machines' clocks must agree on a rate rather than on a value: an executor whose clock runs slow relative to the shard manager's keeps admitting for that fraction of a lease past the shard manager reclaiming it, which is milliseconds under ordinary `ntp` or `chrony` discipline and seconds while such a daemon is slewing off a large offset, so an executor should not be started until its clock is synchronised. An executor whose own copy of the lease has expired stops **admitting** new work for its shards and reports that sharding is not ready, which a worker service retries after refreshing its routing table - invocations already running are not interrupted. A graceful stop - including the `SIGTERM` an orchestrator sends to stop a pod - deregisters the executor instead of waiting for its lease to expire, so its shards move on the next pass. A shard manager restart or failover re-grants the lease of every executor that still answers the startup health check, so a restart never evicts a healthy cluster: only executors that fail that check are removed. An executor whose own copy of the lease lapsed during the outage resumes admitting work on its next successful renewal rather than on the push that follows the re-grant, and neither a re-grant nor a renewal ever moves an executor's deadline earlier, so a `shard_lease_duration` reduced across a restart takes effect only once the new length outruns the deadline each executor was last told. -The lease bounds how long a lost shard keeps being served, but it cannot stop an executor that has already lost one from finishing a write it had started. That is what **oplog fencing** is for. Each agent's oplog records the shard epoch allowed to write it, and every batch of entries is checked against that record inside the same database transaction that inserts them, so a refused write leaves nothing behind. An executor whose epoch is behind is turned away at the storage rather than discovering the problem later: that one agent stops there, and the worker service resumes it on the shard's new owner. The record only ever moves forward - an executor re-granted the shard at a higher epoch takes over, and one holding a stale epoch cannot claim it back - and an absent record fences an open oplog too, because the record is written before an oplog's first entry and removed before its last. That protection covers an oplog whose record exists. Deleting an agent removes its record and forgets the epoch with it: an executor that still has the oplog open is refused, but an oplog without a record - a new agent, a deleted agent created again, or one last written before this release - is claimed by whichever epoch opens it first, so for those only the lease stops an executor that has lost the shard. Ephemeral agents are not fenced: their oplogs are never replayed, so a duplicate there is a duplicated observability record rather than duplicated state. +The lease bounds how long a lost shard keeps being served, but it cannot stop an executor that has already lost one from finishing a write it had started. That is what **oplog fencing** is for. Each agent's oplog records the shard epoch allowed to write it, and every batch of entries is checked against that record inside the same database transaction that inserts them, so a refused write leaves nothing behind. An executor whose epoch is behind is turned away at the storage rather than discovering the problem later: that one agent stops there, and the worker service resumes it on the shard's new owner. The record only ever moves forward - an executor re-granted the shard at a higher epoch takes over, and one holding a stale epoch cannot claim it back - and an absent record fences an open oplog too, because the record is written before an oplog's first entry and removed before its last. That protection covers an oplog whose record exists. Deleting an agent removes its record and forgets the epoch with it: an executor that still has the oplog open is refused, but an oplog without a record - a new agent, a deleted agent created again, or one last written before this release - is claimed by whichever epoch opens it first, so for those only the lease stops an executor that has lost the shard. Ephemeral agents are not fenced: their oplogs are never replayed, so a duplicate there is a duplicated observability record rather than duplicated state. Fencing covers the oplog and nothing else. An executor that has lost a shard can still write that agent's key-value records, its status and its blob payloads, none of which carry an epoch: a stale status blob can overwrite a newer one until the agent's next status change on its new owner, while the oplog - the only record replay reads - stays correct. Nor does the fence undo what has already left the machine: an outgoing call an agent made before its first refused write has happened, and the shard's new owner, having no record of it, makes it again, which is the same at-least-once exposure a crash between the call and its oplog entry already has. A call the executor has not started yet is refused once the fence has latched, so the window is the call in flight rather than every call after the shard moved. -Fencing is enforced by the PostgreSQL and SQLite indexed storage backends. Redis and the in-memory backend cannot enforce it, so an executor configured with one of those **and** a shard manager refuses to start rather than run unprotected; set `indexed_storage.type` to `Postgres`, `Sqlite`, `KVStoreSqlite`, `MultiSqlite` or `KVStoreMultiSqlite`. `KVStoreSqlite` and `KVStoreMultiSqlite` derive their location from the key-value storage, so they also require `key_value_storage.type` to be `Sqlite` or `MultiSqlite` respectively; any other combination fails at startup. The check applies to every executor that registers with a shard manager, whatever its shard count; only an executor that runs without one, such as the debugging service, is exempt. +Fencing is enforced by the PostgreSQL and SQLite indexed storage backends. Redis and the in-memory backend cannot enforce it, so an executor configured with one of those **and** a shard manager refuses to start rather than run unprotected. The shipped configuration still defaults to `KVStoreRedis`, so a distributed deployment that has not set this explicitly fails at startup with that error until it does; set `indexed_storage.type` to `Postgres`, `Sqlite`, `KVStoreSqlite`, `MultiSqlite` or `KVStoreMultiSqlite`. `KVStoreSqlite` and `KVStoreMultiSqlite` derive their location from the key-value storage, so they also require `key_value_storage.type` to be `Sqlite` or `MultiSqlite` respectively; any other combination fails at startup. The check applies to every executor that registers with a shard manager, whatever its shard count; only an executor that runs without one, such as the debugging service, is exempt. -A shard epoch only means something against the shard manager state that minted it. A state that is wiped, replaced or restored from a backup mints epochs below the ones the oplogs already record, and every write at those would be refused, so the shard manager repairs its record from what the executors tell it, without an operator. An executor that kept running sends the epochs it holds on its next renewal, or - when the state no longer lists the executor and refuses that renewal - on the re-registration that follows. The shard manager raises its record to them, and a shard it has meanwhile given to another executor stays with that owner at a new epoch one past the reported one, so two executors never write a shard's oplogs at the same epoch. When every executor restarted as well, none of them holds the old epochs, and the oplogs supply them instead: an executor refused a write reports the epoch the oplog recorded on its next renewal, and the shard manager gives whichever executor owns that shard a new epoch one past it. Either way the affected agents resume at the new epoch within about one renewal interval, while the worker service retries their requests. An oplog nobody opens keeps its old record until it is opened, and that first refused open repairs it the same way. To clear the old records up front instead, or if an agent is still refused after several renewals, stop every executor, run `DELETE FROM oplog_metadata` on the indexed storage - for `MultiSqlite` and `KVStoreMultiSqlite`, on every `*-oplog-*.db` file under the root directory - and start the executors again; each oplog records its owner's epoch when it is next opened. Run the delete only while no executor is running, because an oplog without a record accepts whichever executor opens it first. +A shard epoch only means something against the shard manager state that minted it. A state that is wiped, replaced or restored from a backup mints epochs below the ones the oplogs already record, and every write at those would be refused, so the shard manager repairs its record from what the executors tell it, without an operator. An executor that kept running sends the epochs it holds on its next renewal, or - when the state no longer lists the executor and refuses that renewal - on the re-registration that follows. The shard manager raises its record to them, and a shard it has meanwhile given to another executor stays with that owner at a new epoch one past the reported one, so two executors never write a shard's oplogs at the same epoch. When every executor restarted as well, none of them holds the old epochs, and the oplogs supply them instead: an executor refused a write reports the epoch the oplog recorded on its next renewal, and the shard manager gives whichever executor owns that shard a new epoch one past it. Either way the affected agents resume at the new epoch within about one renewal interval, while the worker service retries their requests. A wiped state also mints from the beginning again, so it can hand an executor the very epoch another one is still writing at. The record names the writing process as well as the epoch, and a second process arriving at an epoch the record already holds is refused rather than sharing it; that refusal is reported on the next renewal like any other, and the shard manager mints its owner one past the collision, so the takeover happens at a generation nobody shares. An oplog nobody opens keeps its old record until it is opened, and that first refused open repairs it the same way. To clear the old records up front instead, or if an agent is still refused after several renewals, stop every executor, run `DELETE FROM oplog_metadata` on the indexed storage - for `MultiSqlite` and `KVStoreMultiSqlite`, on every `*-oplog-*.db` file under the root directory - and start the executors again; each oplog records its owner's epoch when it is next opened. Run the delete only while no executor is running, because an oplog without a record accepts whichever executor opens it first. Upgrade the shard manager and the worker services before the worker executors. An upgraded executor reports a request that reached it without its shard as a routing miss, which only an upgraded worker service retries, and it sends the epochs the repair above relies on in fields an earlier shard manager ignores. Once any executor runs this release, only roll forward: it adds the table that records these epochs to the indexed storage, and an executor from an earlier release refuses to start against indexed storage that has applied that migration, so an older executor restarted mid-rollout, or a rollback of the executor image, fails at startup. During a rolling upgrade the fence is complete only once every executor runs the new code and each agent has been opened once under it. diff --git a/golem-common/src/model/invocation_session_public.rs b/golem-common/src/model/invocation_session_public.rs index 33a6b29643..3dbb4aebca 100644 --- a/golem-common/src/model/invocation_session_public.rs +++ b/golem-common/src/model/invocation_session_public.rs @@ -79,6 +79,11 @@ pub enum PublicErrorCode { ResourceExhausted, ProducerError, InvocationFailed, + /// The executor that answered does not own this agent's shard right now: the assignment is + /// moving, or a write of its was fenced by the shard's new owner. Nothing is wrong with the + /// request, and a client that reconnects reaches the new owner - which is why this is not + /// `InternalError`. + ShardingNotReady, InternalError, } @@ -110,6 +115,7 @@ impl PublicErrorCode { Self::ResourceExhausted => "resource-exhausted", Self::ProducerError => "producer-error", Self::InvocationFailed => "invocation-failed", + Self::ShardingNotReady => "sharding-not-ready", Self::InternalError => "internal-error", } } @@ -164,6 +170,7 @@ const ALL_ERROR_CODES: &[PublicErrorCode] = &[ PublicErrorCode::ResourceExhausted, PublicErrorCode::ProducerError, PublicErrorCode::InvocationFailed, + PublicErrorCode::ShardingNotReady, PublicErrorCode::InternalError, ]; diff --git a/golem-common/src/model/quota.rs b/golem-common/src/model/quota.rs index e5104d8982..2a5811746c 100644 --- a/golem-common/src/model/quota.rs +++ b/golem-common/src/model/quota.rs @@ -55,8 +55,19 @@ impl LeaseEpoch { Self(0) } + /// The next epoch, or `None` when there is none left. + /// + /// The fallible one is the one to use where the counter is advanced: an epoch that cannot + /// advance is a stuck lease, and refusing that one operation keeps the process up, while + /// [`Self::next`]'s panic takes the whole shard manager down and does it again on every retry. + pub fn checked_next(self) -> Option { + self.0.checked_add(1).map(Self) + } + + /// Panics at the ceiling. Only for a caller that has no way to refuse - prefer + /// [`Self::checked_next`]. pub fn next(self) -> Self { - Self(self.0.checked_add(1).expect("LeaseEpoch overflow")) + self.checked_next().expect("LeaseEpoch overflow") } } diff --git a/golem-shard-manager/src/quota/quota_service.rs b/golem-shard-manager/src/quota/quota_service.rs index b7597471e4..1eaa920b70 100644 --- a/golem-shard-manager/src/quota/quota_service.rs +++ b/golem-shard-manager/src/quota/quota_service.rs @@ -214,7 +214,7 @@ impl QuotaService { })?; let snapshot = state.clone(); let prev_rev = state.current_revision(); - let result = state.acquire_lease(pod, self.lease_duration, self.min_executors); + let result = state.acquire_lease(pod, self.lease_duration, self.min_executors)?; if let Err(e) = state.bump_revision() { warn!(error = %e, "failed to bump revision, rolling back"); diff --git a/golem-shard-manager/src/quota/quota_state.rs b/golem-shard-manager/src/quota/quota_state.rs index 25039e24a7..254c56436f 100644 --- a/golem-shard-manager/src/quota/quota_state.rs +++ b/golem-shard-manager/src/quota/quota_state.rs @@ -325,7 +325,7 @@ impl QuotaState { pod: Pod, lease_duration: Duration, min_executors: u64, - ) -> AcquireLeaseResult { + ) -> Result { let expired = self.housekeep(); if let Some(existing) = self.leases.get(&pod) { @@ -353,19 +353,23 @@ impl QuotaState { let pod_lease = self.leases.get_mut(&pod).expect("just inserted"); let epoch = pod_lease.epoch; - pod_lease.epoch = epoch.next(); + pod_lease.epoch = epoch.checked_next().ok_or_else(|| { + QuotaError::InternalError(anyhow::anyhow!( + "lease epoch for pod {pod} cannot advance past {epoch}" + )) + })?; self.remaining -= allocated_amount; pod_lease.allocated = allocated_amount; pod_lease.granted_at = now; pod_lease.expires_at = expires_at; - AcquireLeaseResult { + Ok(AcquireLeaseResult { epoch, allocated_amount, expires_at, expired, total_available_amount, - } + }) } pub fn renew_lease( @@ -406,7 +410,11 @@ impl QuotaState { .get_mut(pod) .expect("just validated and refreshed"); let new_epoch = pod_lease.epoch; - pod_lease.epoch = new_epoch.next(); + pod_lease.epoch = new_epoch.checked_next().ok_or_else(|| { + QuotaError::InternalError(anyhow::anyhow!( + "lease epoch for pod {pod} cannot advance past {new_epoch}" + )) + })?; let allocated_amount = self.compute_allocation(pod, min_executors); let total_available_amount = self.total_available_amount(); diff --git a/golem-shard-manager/src/sharding/model.rs b/golem-shard-manager/src/sharding/model.rs index 8fd9915aaf..607dc4febd 100644 --- a/golem-shard-manager/src/sharding/model.rs +++ b/golem-shard-manager/src/sharding/model.rs @@ -522,7 +522,7 @@ impl ShardLeaseState { executor_id: ExecutorId, claimed: &BTreeMap, ) -> Vec { - self.raise_epoch_floor_for(Some(executor_id), claimed) + self.raise_epoch_floor_for(Some(executor_id), None, claimed) } /// Raises the recorded epoch of every assigned shard `stored` names to one past the stored @@ -544,17 +544,26 @@ impl ShardLeaseState { /// an equal report arrives, and that tie is the equality this cannot tell from a shard move. pub fn raise_epoch_floor_past( &mut self, + reporter: ExecutorId, stored: &BTreeMap, ) -> Vec { - self.raise_epoch_floor_for(None, stored) + self.raise_epoch_floor_for(None, Some(reporter), stored) } /// The one rule behind [`Self::raise_epoch_floor`] and [`Self::raise_epoch_floor_past`]: an /// entry `holder` owns takes the epoch itself, any other assigned entry is minted one past it, /// and an unassigned shard's high-water takes it. `None` holds nothing. + /// + /// `reporter` is the executor a fenced report came from, and only [`Self::raise_epoch_floor_past`] + /// has one. It settles the single case equality cannot: a report of the epoch this shard is + /// already recorded at, from the executor currently assigned that shard, means the storage + /// refused the assignee at its own generation - so somebody else holds it. Only a manager that + /// lost its state mints one generation twice, and the repair is to mint past it. Every other + /// report at or below the record is the ordinary loser of a shard move and moves nothing. fn raise_epoch_floor_for( &mut self, holder: Option, + reporter: Option, claimed: &BTreeMap, ) -> Vec { let mut raised = Vec::new(); @@ -564,13 +573,25 @@ impl ShardLeaseState { if !self.contains_shard(*shard_id) { continue; } + // The assignee reporting a fence at the epoch it was granted: the row refused the + // executor the manager believes owns it, so another writer holds that generation. + let collides_with_the_assignee = reporter.is_some_and(|reporter| { + self.shard_epochs + .get(shard_id) + .is_some_and(|recorded| recorded == claimed_epoch) + && self + .shard_assignments + .get(shard_id) + .is_some_and(|entry| entry.executor_id == reporter) + }); // Against the high-water, so the floor only ever climbs: a value at or below one this // shard has already reached is not evidence of anything. On a shard somebody else // owns, that is the ordinary missed push, and the grant corrects it. - if self - .shard_epochs - .get(shard_id) - .is_some_and(|recorded| recorded >= claimed_epoch) + if !collides_with_the_assignee + && self + .shard_epochs + .get(shard_id) + .is_some_and(|recorded| recorded >= claimed_epoch) { continue; } @@ -580,10 +601,11 @@ impl ShardLeaseState { // clears the claimant's oplog rows. An unassigned shard has no entry to corrupt, and // raising its high-water only makes the next mint start above the epoch the claim // proves is already in use. - let mints_past_claim = self - .shard_assignments - .get(shard_id) - .is_some_and(|entry| Some(entry.executor_id) != holder); + let mints_past_claim = collides_with_the_assignee + || self + .shard_assignments + .get(shard_id) + .is_some_and(|entry| Some(entry.executor_id) != holder); // The guard is on the *candidate* - what this would actually store - not on // `claimed_epoch` itself: a claim of `u64::MAX` overflows `checked_next` right here, // but a claim of `u64::MAX - 1` does not, and minting past it succeeds, landing @@ -618,21 +640,45 @@ impl ShardLeaseState { /// the owner stays the same, one past the highest epoch ever recorded for that shard when the /// owner changes. /// + /// `None` when there is no epoch left above the recorded one. Fallible rather than panicking + /// because the recorded value is not this process's to trust: it can be raised from a wire + /// claim or a fenced report (see [`Self::raise_epoch_floor_for`], which drops a candidate + /// landing on `u64::MAX` for the same reason), and a shard whose epoch cannot advance must + /// stay where it is rather than abort the manager on every retry of the same plan. + /// /// Pure, and the single definition of the rule; [`Self::assign_shard`] mints with it. - pub fn next_epoch_for(&self, executor_id: ExecutorId, shard_id: ShardId) -> ShardEpoch { + pub fn next_epoch_for(&self, executor_id: ExecutorId, shard_id: ShardId) -> Option { match self.shard_assignments.get(&shard_id) { - Some(entry) if entry.executor_id == executor_id => entry.epoch, + Some(entry) if entry.executor_id == executor_id => Some(entry.epoch), + // The `u64::MAX` filter is the same stance as `raise_epoch_floor_for`'s, and for the + // same reason: storing it would leave the *next* owner change with no epoch to mint, + // so the ceiling is refused one generation earlier, while the shard can still be left + // where it is. _ => match self.shard_epochs.get(&shard_id) { - Some(last) => last.next(), - None => ShardEpoch::initial(), + Some(last) => last.checked_next().filter(|epoch| epoch.0 != u64::MAX), + None => Some(ShardEpoch::initial()), }, } } - pub fn assign_shard(&mut self, executor_id: ExecutorId, shard_id: ShardId) -> ShardEpoch { - let epoch = self.next_epoch_for(executor_id, shard_id); + /// Assigns `shard_id` to `executor_id` at a freshly minted epoch, or leaves it alone when the + /// epoch cannot advance. An unassigned shard is re-homed by the next plan; a panic here would + /// take the manager down and do it again on the next attempt. + pub fn assign_shard( + &mut self, + executor_id: ExecutorId, + shard_id: ShardId, + ) -> Option { + let Some(epoch) = self.next_epoch_for(executor_id, shard_id) else { + warn!( + shard_id = %shard_id, + executor_id = %executor_id, + "Refusing to assign a shard whose epoch cannot advance; leaving it unassigned" + ); + return None; + }; self.assign_shard_with_epoch(executor_id, shard_id, epoch); - epoch + Some(epoch) } /// Records `epoch` for `shard_id`. [`Self::assign_shard`] is the only caller, and mints the @@ -1034,6 +1080,13 @@ mod tests { ExecutorId(Uuid::from_u128(idx)) } + /// An executor that holds nothing, for tests that only care about the epochs a fenced report + /// carries. The reporter matters only when it is the shard's current assignee - the collision + /// case, which `a_fence_reported_by_the_assignee_at_its_own_epoch_is_minted_past` covers. + fn reporting_executor() -> ExecutorId { + executor(9999) + } + fn addr(idx: u8) -> ExecutorAddr { ExecutorAddr { ip: IpAddr::V4(Ipv4Addr::new(10, 0, 0, idx)), @@ -1193,20 +1246,20 @@ mod tests { // first assignment starts at the initial epoch assert_eq!( shard_state.assign_shard(executor(1), shard(0)), - ShardEpoch::initial() + Some(ShardEpoch::initial()) ); assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); // re-assigning to the same owner is idempotent assert_eq!( shard_state.assign_shard(executor(1), shard(0)), - ShardEpoch(0) + Some(ShardEpoch(0)) ); // moving to another owner advances the epoch; stored == returned assert_eq!( shard_state.assign_shard(executor(2), shard(0)), - ShardEpoch(1) + Some(ShardEpoch(1)) ); assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); assert_eq!( @@ -1228,7 +1281,7 @@ mod tests { ); assert_eq!( shard_state.assign_shard(executor(1), shard(0)), - ShardEpoch(2) + Some(ShardEpoch(2)) ); assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(2))); } @@ -1246,18 +1299,18 @@ mod tests { // ... and later handed to other executors with epochs the evicted owner never held assert_eq!( shard_state.assign_shard(executor(2), shard(0)), - ShardEpoch(1) + Some(ShardEpoch(1)) ); assert_eq!( shard_state.assign_shard(executor(3), shard(1)), - ShardEpoch(1) + Some(ShardEpoch(1)) ); // a second eviction keeps advancing shard_state.remove_executor(executor(2)); assert_eq!( shard_state.assign_shard(executor(3), shard(0)), - ShardEpoch(2) + Some(ShardEpoch(2)) ); // housekeep-driven eviction behaves the same way @@ -1269,11 +1322,11 @@ mod tests { shard_state.add_executor(executor(4), addr(4), None, expired, TTL); assert_eq!( shard_state.assign_shard(executor(4), shard(0)), - ShardEpoch(3) + Some(ShardEpoch(3)) ); assert_eq!( shard_state.assign_shard(executor(4), shard(1)), - ShardEpoch(2) + Some(ShardEpoch(2)) ); assert!(shard_state.check_invariants().is_ok()); } @@ -1312,7 +1365,7 @@ mod tests { shard_state.add_executor(executor(2), addr(2), None, t0(), TTL); assert_eq!( shard_state.assign_shard(executor(2), shard(0)), - ShardEpoch(6) + Some(ShardEpoch(6)) ); assert!(shard_state.check_invariants().is_ok()); } @@ -1369,7 +1422,7 @@ mod tests { assert!(shard_state.check_invariants().is_ok()); assert_eq!( shard_state.assign_shard(executor(2), shard(2)), - ShardEpoch(5) + Some(ShardEpoch(5)) ); } @@ -1405,7 +1458,7 @@ mod tests { let fenced_at_max = BTreeMap::from([(shard(1), ShardEpoch(u64::MAX))]); assert!( shard_state - .raise_epoch_floor_past(&fenced_at_max) + .raise_epoch_floor_past(reporting_executor(), &fenced_at_max) .is_empty(), "a fenced epoch at u64::MAX must be ignored, not minted past" ); @@ -1451,7 +1504,7 @@ mod tests { shard_state.add_executor(executor(3), addr(3), None, t0(), TTL); assert_eq!( shard_state.assign_shard(executor(3), shard(0)), - ShardEpoch(1), + Some(ShardEpoch(1)), "an ordinary reassignment after the rejected claim mints normally, one past the \ epoch that was never disturbed" ); @@ -1462,7 +1515,7 @@ mod tests { let fenced_near_max = BTreeMap::from([(shard(1), ShardEpoch(u64::MAX - 1))]); assert!( shard_state - .raise_epoch_floor_past(&fenced_near_max) + .raise_epoch_floor_past(reporting_executor(), &fenced_near_max) .is_empty(), "a fenced epoch of u64::MAX - 1 must be ignored for the same reason" ); @@ -1533,6 +1586,74 @@ mod tests { assert!(shard_state.check_invariants().is_ok()); } + #[test] + fn a_fence_reported_by_the_assignee_at_its_own_epoch_is_minted_past() { + // The wiped-store collision. Executor 1 is assigned shard 0 at epoch 0 - all a fresh state + // can mint - and its write is refused by a row another executor still holds at that same + // epoch. Equality is ordinarily the loser of a shard move and moves nothing, but the + // reporter here IS the assignee: the storage refused the executor this state believes owns + // the shard, so somebody else holds the generation and it has to be minted past. + let mut shard_state = shard_state_with(4, &[(1, 1, &[0])]); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(0))); + + let fenced = BTreeMap::from([(shard(0), ShardEpoch(0))]); + assert_eq!( + shard_state.raise_epoch_floor_past(executor(1), &fenced), + vec![shard(0)] + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); + assert!(shard_state.check_invariants().is_ok()); + } + + #[test] + fn a_fence_reported_at_the_record_by_anyone_else_moves_nothing() { + // The ordinary shard move, which must not churn: executor 2 took shard 0 over at epoch 1 + // and recorded it, so executor 1's refused write reports 1 while the state already says 1. + // The reporter is not the assignee, so this is the loser of the move, not a collision. + let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[])]); + shard_state.assign_shard(executor(2), shard(0)); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); + + let fenced = BTreeMap::from([(shard(0), ShardEpoch(1))]); + assert!( + shard_state + .raise_epoch_floor_past(executor(1), &fenced) + .is_empty() + ); + assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(1))); + } + + #[test] + fn a_shard_whose_epoch_cannot_advance_is_left_unassigned_rather_than_panicking() { + // A near-ceiling epoch can be planted by a wire claim (see the `u64::MAX` guard in + // `raise_epoch_floor_for`). The next owner change then has no epoch to mint, and the shard + // stays where it is: the next plan re-homes it, while a panic would take the manager down + // and do it again on every retry. + let mut shard_state = shard_state_with(4, &[(1, 1, &[0]), (2, 2, &[])]); + // Planted the way production can: a fenced report carries a near-ceiling epoch, and the + // floor rises to it on the shard's own owner. + shard_state.raise_epoch_floor( + executor(1), + &BTreeMap::from([(shard(0), ShardEpoch(u64::MAX - 1))]), + ); + assert_eq!( + shard_state.epoch_for_shard(shard(0)), + Some(ShardEpoch(u64::MAX - 1)) + ); + + assert_eq!(shard_state.next_epoch_for(executor(2), shard(0)), None); + assert_eq!(shard_state.assign_shard(executor(2), shard(0)), None); + assert_eq!( + shard_state + .shard_assignments + .get(&shard(0)) + .map(|entry| entry.executor_id), + Some(executor(1)), + "the shard keeps its owner rather than moving to an epoch that does not exist" + ); + assert!(shard_state.check_invariants().is_ok()); + } + #[test] fn a_fenced_epoch_re_mints_every_owner_above_it_the_reporters_own_included() { // Writes were refused by rows at epoch 3 on shards 0, 1 and 2, in a state that lost @@ -1546,7 +1667,7 @@ mod tests { (shard(2), ShardEpoch(3)), ]); assert_eq!( - shard_state.raise_epoch_floor_past(&stored), + shard_state.raise_epoch_floor_past(reporting_executor(), &stored), vec![shard(0), shard(1), shard(2)] ); for (shard_id, owner) in [(0, 1), (1, 2)] { @@ -1575,16 +1696,23 @@ mod tests { ); assert_eq!( shard_state.next_epoch_for(executor(1), shard(2)), - ShardEpoch(4) + Some(ShardEpoch(4)) ); assert!(shard_state.check_invariants().is_ok()); // Reported again, every epoch is below the record: nothing moves a second time. - assert!(shard_state.raise_epoch_floor_past(&stored).is_empty()); + assert!( + shard_state + .raise_epoch_floor_past(reporting_executor(), &stored) + .is_empty() + ); // At the record is the ordinary loser of a shard move. assert!( shard_state - .raise_epoch_floor_past(&BTreeMap::from([(shard(0), ShardEpoch(4))])) + .raise_epoch_floor_past( + reporting_executor(), + &BTreeMap::from([(shard(0), ShardEpoch(4))]) + ) .is_empty() ); assert_eq!(shard_state.epoch_for_shard(shard(0)), Some(ShardEpoch(4))); @@ -1601,11 +1729,11 @@ mod tests { let claimed = BTreeMap::from([(shard(0), ShardEpoch(claimed))]); let fenced = BTreeMap::from([(shard(0), ShardEpoch(fenced))]); if fenced_first { - shard_state.raise_epoch_floor_past(&fenced); + shard_state.raise_epoch_floor_past(reporting_executor(), &fenced); shard_state.raise_epoch_floor(executor(claimant), &claimed); } else { shard_state.raise_epoch_floor(executor(claimant), &claimed); - shard_state.raise_epoch_floor_past(&fenced); + shard_state.raise_epoch_floor_past(reporting_executor(), &fenced); } assert!(shard_state.check_invariants().is_ok()); assert_eq!( diff --git a/golem-shard-manager/src/sharding/shard_management.rs b/golem-shard-manager/src/sharding/shard_management.rs index b87bdaf03c..8a734685f0 100644 --- a/golem-shard-manager/src/sharding/shard_management.rs +++ b/golem-shard-manager/src/sharding/shard_management.rs @@ -443,7 +443,7 @@ impl ShardManagement { // applied first, a fenced epoch at or above the claim ends one past it, and a higher // claim still wins. They reach a state that was wiped or replaced as well, because the // executor keeps reporting them until a renewal under its re-registered id is granted. - let re_minted = shard_state.raise_epoch_floor_past(&fenced); + let re_minted = shard_state.raise_epoch_floor_past(executor_id, &fenced); let fence_re_minted_owners = owners_re_minted_by(shard_state, &re_minted, executor_id); if !re_minted.is_empty() { warn!( diff --git a/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql b/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql index 54d48ad904..7bbd2841ba 100644 --- a/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql +++ b/golem-worker-executor/db/migration/indexed/postgres/002_oplog_metadata.sql @@ -1,13 +1,21 @@ --- The shard epoch authorised to write each oplog. +-- The shard epoch authorised to write each oplog, and the writer holding it. -- -- One row per key in `index_storage`, carrying the ownership generation of the shard the agent -- belongs to. An append asserts its epoch against this row inside the same transaction as the -- insert, so an executor that has lost the shard cannot keep writing to an oplog whose owner has -- moved on. A missing row fences too: the row is written before the first entry, and removed -- before the entries are. +-- +-- `owner` is the writing process, not the executor's lease identity, which is regenerated whenever +-- the shard manager answers `LeaseNotFound`. The epoch alone cannot separate two writers holding +-- the same number, which a shard manager that lost its state hands out when it mints from zero +-- again: the process that recorded the row is the one allowed to go on writing at that epoch, and +-- any other writer is refused and reports it, so the manager mints above and the takeover happens +-- at a generation nobody shares. CREATE TABLE oplog_metadata ( namespace TEXT NOT NULL, key TEXT NOT NULL, epoch BIGINT NOT NULL, + owner TEXT NOT NULL, PRIMARY KEY (namespace, key) ); diff --git a/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql b/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql index 8aa054dfb7..8e2005e038 100644 --- a/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql +++ b/golem-worker-executor/db/migration/indexed/sqlite/002_oplog_metadata.sql @@ -1,7 +1,9 @@ --- The shard epoch authorised to write each oplog. See the postgres migration of the same name. +-- The shard epoch authorised to write each oplog, and the writer holding it. See the postgres +-- migration of the same name. CREATE TABLE oplog_metadata ( namespace TEXT NOT NULL, key TEXT NOT NULL, epoch INTEGER NOT NULL, + owner TEXT NOT NULL, PRIMARY KEY (namespace, key) ); diff --git a/golem-worker-executor/src/durable_host/concurrent/tests.rs b/golem-worker-executor/src/durable_host/concurrent/tests.rs index 082dc7cf26..25ade75146 100644 --- a/golem-worker-executor/src/durable_host/concurrent/tests.rs +++ b/golem-worker-executor/src/durable_host/concurrent/tests.rs @@ -474,6 +474,7 @@ async fn a_fenced_completion_marker_is_reported_rather_than_panicked() { agent_id: agent_id.clone(), expected_epoch: golem_common::model::ShardEpoch(3), actual_epoch: Some(golem_common::model::ShardEpoch(4)), + owner_conflict: false, })); let seed_oplog = Arc::new(InMemoryOplog::new()); seed_oplog diff --git a/golem-worker-executor/src/durable_host/durable_session/tests.rs b/golem-worker-executor/src/durable_host/durable_session/tests.rs index c1a9dde2f1..f17c4b22f2 100644 --- a/golem-worker-executor/src/durable_host/durable_session/tests.rs +++ b/golem-worker-executor/src/durable_host/durable_session/tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::durable_host::durable_stream::AttachedStreamSegmentSource; use crate::durable_host::durable_stream::tests::{ - TestIdentity, TestOplog, attachment_key, identity, registration, + TestIdentity, TestOplog, attachment_key, identity, registration, test_fence, }; use crate::durable_host::stream_transport::{output_stream_pair, test_output_stream_pair}; use crate::services::oplog::CommitLevel; @@ -1509,6 +1509,147 @@ async fn foreign_preparation_releases_local_owner_when_rpc_requests_retirement() retirement_from_foreign_rpc(true).await; } +/// A foreign mapping prepared while this agent's oplog is already fenced must fail as a fence and +/// leave the session unlocked. +/// +/// Activating a foreign mapping is one of the paths that runs while a shard is being taken away: +/// it writes a session record, so a latched fence has to stop it rather than let it record a +/// mapping the new owner will never see. The session lock matters as much as the error - a +/// preparation that failed while holding it would strand every later call on this session, and the +/// agent is about to be given up, not restarted. +#[test] +#[test_r::timeout("15s")] +async fn a_fenced_oplog_refuses_a_foreign_mapping_and_releases_the_session() { + let local = identity(); + let mut remote = identity(); + remote.agent_id.agent_id.push_str("-remote"); + remote.invocation.callee = remote.agent_id.clone(); + let remote_producer = DurableStreamStore::load( + Arc::new(TestOplog::default()), + remote.environment_id, + remote.agent_id.clone(), + remote.fingerprint, + None, + ) + .await + .unwrap(); + let handle = remote_producer + .register( + None, + registration( + &remote, + StreamRegistrationCoordinate::Root { + invocation_id: remote.invocation.clone(), + root_kind: StreamRootKind::MethodResult, + recursive_value_path: Vec::new(), + }, + StreamSourceKind::InvocationOutput, + ), + ) + .await + .unwrap() + .value; + + let oplog = Arc::new(TestOplog::default()); + let producer = DurableStreamStore::load( + oplog.clone(), + local.environment_id, + local.agent_id.clone(), + local.fingerprint, + None, + ) + .await + .unwrap(); + let attachment_id = AttachmentId::primary( + local.environment_id, + &local.agent_id, + &local.invocation.idempotency_key, + ) + .unwrap(); + let attempt_id = AttemptId::fresh(); + let pending_invocation_oplog_index = append_prepared_pending( + &producer, + &oplog, + &local, + attachment_id, + attempt_id, + &handle, + SessionStreamRole::Input, + ) + .await; + producer + .append_session_record( + None, + StreamSessionRecord::Attached(StreamSessionAttachedRecord { + format_version: DURABLE_STREAM_FORMAT_VERSION, + session_key: local.invocation.clone(), + attachment_id, + attempt_id, + epoch: 1, + pending_invocation_oplog_index, + }), + ) + .await + .unwrap(); + let streams = StreamSession::new( + producer.clone(), + oplog.clone(), + local.invocation.clone(), + [StreamSessionMappingRecord { + transport_stream_id: 7, + handle: handle.clone(), + role: SessionStreamRole::Input, + }], + ) + .with_consumer_journal(Arc::new(TestConsumerJournal(oplog.clone()))) + .with_attachment(1, attempt_id) + .with_rpc(Arc::new(AttachedProducerRpc { + producer: remote_producer, + cancellation_owner: None, + stall_next_cancel: Default::default(), + scripted_reads: Mutex::default(), + pending_read: Mutex::default(), + read_requests: Mutex::default(), + })) + .with_auth_ctx(AuthCtx::System); + + let committed_before = oplog.committed_length(); + // Both halves of what a real fenced oplog does: the latch every reader consults, and the + // refusal an add itself gets once it has latched (`PrimaryOplogState::refuse_if_fenced`). + oplog.latch_fence(test_fence()); + oplog.refuse_adds(test_fence()); + + let error = tokio::time::timeout( + Duration::from_secs(5), + streams.prepare_foreign_mapping( + StreamSessionMappingRecord { + transport_stream_id: 7, + handle, + role: SessionStreamRole::Input, + }, + 1, + ), + ) + .await + .expect("a foreign mapping on a fenced oplog hung instead of failing") + .unwrap_err(); + + assert!( + error.contains("Fenced"), + "a foreign mapping on a fenced oplog must report the fence itself, so the caller reroutes \ + instead of treating it as a local failure, got {error}" + ); + assert_eq!( + oplog.committed_length(), + committed_before, + "nothing may be committed for a mapping the storage refused" + ); + assert!( + streams.session_lock.try_lock().is_ok(), + "the session lock has to be released, or every later call on this session strands" + ); +} + async fn retirement_from_foreign_rpc(prepare: bool) { let local = identity(); let mut remote = identity(); diff --git a/golem-worker-executor/src/durable_host/durable_stream/tests.rs b/golem-worker-executor/src/durable_host/durable_stream/tests.rs index d87f41650f..d56c2f86cb 100644 --- a/golem-worker-executor/src/durable_host/durable_stream/tests.rs +++ b/golem-worker-executor/src/durable_host/durable_stream/tests.rs @@ -94,7 +94,7 @@ impl TestOplog { std::mem::take(&mut *self.read_ranges.lock().unwrap()) } - fn committed_length(&self) -> u64 { + pub(crate) fn committed_length(&self) -> u64 { self.state.lock().unwrap().committed.as_u64() } @@ -112,11 +112,11 @@ impl TestOplog { .collect() } - fn refuse_adds(&self, fence: crate::services::oplog::OplogFence) { + pub(crate) fn refuse_adds(&self, fence: crate::services::oplog::OplogFence) { self.state.lock().unwrap().refused_adds = Some(fence); } - fn latch_fence(&self, fence: crate::services::oplog::OplogFence) { + pub(crate) fn latch_fence(&self, fence: crate::services::oplog::OplogFence) { self.state.lock().unwrap().fence = Some(fence); } } @@ -5919,11 +5919,12 @@ async fn session_control_batch_validates_before_appending_any_record() { } } -fn test_fence() -> crate::services::oplog::OplogFence { +pub(crate) fn test_fence() -> crate::services::oplog::OplogFence { crate::services::oplog::OplogFence { agent_id: identity().agent_id, expected_epoch: golem_common::model::ShardEpoch(3), actual_epoch: Some(golem_common::model::ShardEpoch(4)), + owner_conflict: false, } } diff --git a/golem-worker-executor/src/durable_host/mod.rs b/golem-worker-executor/src/durable_host/mod.rs index 7828d46f5c..1d23aaf72b 100644 --- a/golem-worker-executor/src/durable_host/mod.rs +++ b/golem-worker-executor/src/durable_host/mod.rs @@ -3078,6 +3078,25 @@ impl DurableWorkerCtx { self.state.current_retry_point = result; Ok(result) } else { + // No scope opens, so nothing is written before the side effect runs and a fence + // already latched would only surface at the commit after it - by which time the + // effect has happened and the shard's new owner, having no record of it, runs it + // again. Reading the latch costs no storage round trip, so a write whose effect is + // about to run is refused here instead. It does not close the window where the shard + // moves *during* the call: that one needs a round trip per call, which is exactly what + // an idempotent write is declared to avoid. + if self.state.is_live() + && matches!( + function_type, + DurableFunctionType::WriteRemote + | DurableFunctionType::WriteRemoteBatched(_) + | DurableFunctionType::WriteRemoteTransaction(_) + ) + && let Some(fence) = self.state.oplog.fence() + { + return Err(WorkerExecutorError::from(OplogError::Fenced(fence))); + } + // When there is no scope `Start` entry, the current retry point can only // point to the last written non-hint entry. Hint entries must be ignored // because they are nondeterministic. diff --git a/golem-worker-executor/src/durable_host/p3/http/send.rs b/golem-worker-executor/src/durable_host/p3/http/send.rs index 83d838fab2..e2ad60f95e 100644 --- a/golem-worker-executor/src/durable_host/p3/http/send.rs +++ b/golem-worker-executor/src/durable_host/p3/http/send.rs @@ -1553,6 +1553,7 @@ mod tests { }, expected_epoch: ShardEpoch(8), actual_epoch: Some(ShardEpoch(9)), + owner_conflict: false, }); record_frame_entry( oplog.clone(), diff --git a/golem-worker-executor/src/model/mod.rs b/golem-worker-executor/src/model/mod.rs index b22690a269..1de06c1560 100644 --- a/golem-worker-executor/src/model/mod.rs +++ b/golem-worker-executor/src/model/mod.rs @@ -961,6 +961,7 @@ mod tests { }, expected_epoch: golem_common::model::ShardEpoch(7), actual_epoch: Some(golem_common::model::ShardEpoch(8)), + owner_conflict: false, }; let trap = TrapType::from_error::( @@ -1008,6 +1009,7 @@ mod tests { }, expected_epoch: golem_common::model::ShardEpoch(7), actual_epoch: Some(golem_common::model::ShardEpoch(8)), + owner_conflict: false, } } diff --git a/golem-worker-executor/src/services/oplog/mod.rs b/golem-worker-executor/src/services/oplog/mod.rs index 17549655a9..9fda226afa 100644 --- a/golem-worker-executor/src/services/oplog/mod.rs +++ b/golem-worker-executor/src/services/oplog/mod.rs @@ -518,6 +518,10 @@ pub struct OplogFence { pub agent_id: AgentId, pub expected_epoch: ShardEpoch, pub actual_epoch: Option, + /// The stored epoch is the one this executor asserted, but another process recorded it. The + /// epoch alone therefore says nothing about who may write, and the shard manager has to mint + /// past it rather than leave two holders on one generation. + pub owner_conflict: bool, } /// Told of every refusal the storage returns, carrying the epoch recorded on the oplog. diff --git a/golem-worker-executor/src/services/oplog/plugin.rs b/golem-worker-executor/src/services/oplog/plugin.rs index 310956d6cd..4f1ceaf75c 100644 --- a/golem-worker-executor/src/services/oplog/plugin.rs +++ b/golem-worker-executor/src/services/oplog/plugin.rs @@ -3326,6 +3326,7 @@ mod tests { agent_id: metadata.agent_id.clone(), expected_epoch: ShardEpoch(1), actual_epoch: Some(ShardEpoch(2)), + owner_conflict: false, }); let inner: Arc = in_memory.clone(); diff --git a/golem-worker-executor/src/services/oplog/primary.rs b/golem-worker-executor/src/services/oplog/primary.rs index 4ccf68974a..9ca4915440 100644 --- a/golem-worker-executor/src/services/oplog/primary.rs +++ b/golem-worker-executor/src/services/oplog/primary.rs @@ -350,18 +350,23 @@ async fn record_owning_epoch( match outcome { Ok(()) => None, Err(IndexedStorageError::Fenced { - expected, actual, .. + expected, + actual, + owner_conflict, + .. }) => { warn!( agent_id = %owned_agent_id, expected_epoch = expected.0, actual_epoch = ?actual.map(|epoch| epoch.0), + owner_conflict, "Oplog opened at a stale shard epoch: the shard has a new owner" ); let fence = OplogFence { agent_id: owned_agent_id.agent_id(), expected_epoch: expected, actual_epoch: actual, + owner_conflict, }; if let Some(observer) = fence_observer { observer.fenced(&fence); @@ -2110,11 +2115,15 @@ impl PrimaryOplogState { fn as_oplog_error(owned_agent_id: &OwnedAgentId, err: IndexedStorageError) -> OplogError { match err { IndexedStorageError::Fenced { - expected, actual, .. + expected, + actual, + owner_conflict, + .. } => OplogError::Fenced(OplogFence { agent_id: owned_agent_id.agent_id(), expected_epoch: expected, actual_epoch: actual, + owner_conflict, }), other => OplogError::Storage(other.to_string()), } diff --git a/golem-worker-executor/src/services/oplog/tests.rs b/golem-worker-executor/src/services/oplog/tests.rs index f3e497b44f..28153fad2d 100644 --- a/golem-worker-executor/src/services/oplog/tests.rs +++ b/golem-worker-executor/src/services/oplog/tests.rs @@ -515,6 +515,7 @@ impl InjectedAppendFailure { key: key.to_string(), expected: shard_epoch.unwrap_or_default(), actual: shard_epoch.map(|epoch| ShardEpoch(epoch.0 + 1)), + owner_conflict: false, }), _ => None, } @@ -9507,6 +9508,7 @@ async fn a_refused_open_or_create_reports_the_stored_epoch_to_the_fence_observer agent_id: agent_id.clone(), expected_epoch: ShardEpoch(5), actual_epoch: Some(ShardEpoch(6)), + owner_conflict: false, }; for agent_id in [&opened, &created] { @@ -9669,6 +9671,7 @@ async fn a_create_refused_behind_a_cached_handle_still_reports_the_stored_epoch( agent_id: agent_id.clone(), expected_epoch: ShardEpoch(5), actual_epoch: Some(ShardEpoch(6)), + owner_conflict: false, }] ); } @@ -9729,6 +9732,7 @@ async fn a_refused_append_reports_the_stored_epoch_and_the_latch_does_not_report agent_id: agent_id.clone(), expected_epoch: ShardEpoch(5), actual_epoch: Some(ShardEpoch(6)), + owner_conflict: false, }; assert_eq!( recorder.fences(), diff --git a/golem-worker-executor/src/services/shard.rs b/golem-worker-executor/src/services/shard.rs index 615c745b43..b17d1daa14 100644 --- a/golem-worker-executor/src/services/shard.rs +++ b/golem-worker-executor/src/services/shard.rs @@ -327,10 +327,14 @@ impl OplogFenceObserver for ShardServiceDefault { // Only a record ahead of the epoch this executor asserted says anything the shard manager // may have lost. An absent record carries no epoch, and one at or below the assertion is // not a generation above it. - let Some(stored) = fence - .actual_epoch - .filter(|stored| *stored > fence.expected_epoch) - else { + // + // The exception is a record at the assertion held by another writer: that one says the + // manager handed the same generation to two executors, which only a manager that lost its + // state does, and it has to mint past the epoch rather than leave it shared. + let Some(stored) = fence.actual_epoch.filter(|stored| { + *stored > fence.expected_epoch + || (fence.owner_conflict && *stored == fence.expected_epoch) + }) else { return; }; let shard_id = { @@ -454,6 +458,7 @@ mod tests { agent_id: agent_id.clone(), expected_epoch: ShardEpoch(expected), actual_epoch: actual.map(ShardEpoch), + owner_conflict: false, } } diff --git a/golem-worker-executor/src/services/shard_manager.rs b/golem-worker-executor/src/services/shard_manager.rs index d1e99d9ceb..f7f27d89e7 100644 --- a/golem-worker-executor/src/services/shard_manager.rs +++ b/golem-worker-executor/src/services/shard_manager.rs @@ -2111,6 +2111,7 @@ mod tests { agent_id: agent_on_shard(shard), expected_epoch: ShardEpoch(expected), actual_epoch: Some(ShardEpoch(stored)), + owner_conflict: false, } } diff --git a/golem-worker-executor/src/storage/indexed/mod.rs b/golem-worker-executor/src/storage/indexed/mod.rs index c93ac6b511..2f683675a2 100644 --- a/golem-worker-executor/src/storage/indexed/mod.rs +++ b/golem-worker-executor/src/storage/indexed/mod.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::fmt::{self, Debug, Display, Formatter}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::time::Duration; use async_trait::async_trait; @@ -23,6 +23,7 @@ use golem_common::model::agent::AgentMode; use golem_common::model::{AgentId, ShardEpoch}; use golem_common::serialization::{deserialize, serialize}; use golem_service_base::repo::RepoError; +use uuid::Uuid; pub mod memory; pub mod multi_sqlite; @@ -47,7 +48,8 @@ pub enum IndexedStorageError { /// Permanent error — data issue or schema error. Caller should not retry. Other(String), /// The write was refused because the writer no longer owns the agent's shard: the epoch it - /// asserted is behind the one recorded for that oplog. + /// asserted is behind the one recorded for that oplog, or the record is held by another + /// writer at the same epoch. /// /// Never retriable — retrying cannot make this executor the owner again. It is not a failure /// of the storage either: the write was rejected on purpose, by a newer owner's claim. @@ -55,9 +57,42 @@ pub enum IndexedStorageError { key: String, expected: ShardEpoch, actual: Option, + /// The stored epoch equals the asserted one but another writer recorded it. Only a shard + /// manager that lost its state mints a generation somebody already holds, so this says + /// the epoch itself has to be minted past - see [`WriterId`]. + owner_conflict: bool, }, } +/// The process behind an oplog write, recorded alongside the epoch it asserts. +/// +/// One value per executor process, kept for the life of the process. It is deliberately *not* the +/// executor's lease identity (`GrpcShardManagerService::executor_id`), which is regenerated +/// whenever the manager answers `LeaseNotFound`: that identity changes while the process goes on +/// holding the same epochs for the same oplogs, and a row keyed on it would refuse the process its +/// own agents after every re-registration. +/// +/// What it buys is the one thing an epoch cannot say by itself: which of two writers holding the +/// same number wrote the record. A manager whose state was wiped mints from zero again and can +/// grant a live owner's epoch to somebody else; both would then pass an equality check. With the +/// writer recorded, the newcomer is refused, reports the collision, and the manager mints above it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct WriterId(pub Uuid); + +impl WriterId { + /// This process's writer identity, created once on first use. + pub fn process() -> Self { + static PROCESS: OnceLock = OnceLock::new(); + *PROCESS.get_or_init(|| WriterId(Uuid::new_v4())) + } +} + +impl Display for WriterId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + impl IndexedStorageError { pub fn is_retriable(&self) -> bool { matches!(self, IndexedStorageError::Transient(_)) @@ -77,7 +112,13 @@ impl Display for IndexedStorageError { key, expected, actual, + owner_conflict, } => match actual { + Some(actual) if *owner_conflict => write!( + f, + "Oplog write fenced for key {key}: asserted shard epoch {expected}, \ + which another writer holds - the stored epoch is {actual}" + ), Some(actual) => write!( f, "Oplog write fenced for key {key}: asserted shard epoch {expected}, \ @@ -117,7 +158,11 @@ pub(crate) enum FencedTxError { key: String, expected: ShardEpoch, actual: Option, + owner_conflict: bool, }, + /// A stored value the schema should have made impossible - a negative epoch, say. Not a fence: + /// nobody took the oplog over, the row itself cannot be trusted. + Corrupt(String), } impl From for FencedTxError { @@ -138,11 +183,14 @@ impl FencedTxError { key, expected, actual, + owner_conflict, } => IndexedStorageError::Fenced { key, expected, actual, + owner_conflict, }, + FencedTxError::Corrupt(msg) => IndexedStorageError::Other(msg), } } } @@ -378,17 +426,21 @@ pub trait IndexedStorage: Debug + Sync { last_dropped_id: u64, ) -> Result<(), IndexedStorageError>; - /// Records the shard epoch that is authorised to write the given key, as a monotonic - /// compare-and-set: the write is accepted when `shard_epoch` is at least the stored one, and - /// refused with [`IndexedStorageError::Fenced`] when it is behind. Inserts the record if the - /// key has none. + /// Records the shard epoch that is authorised to write the given key, and this process as its + /// writer, as a monotonic compare-and-set: the write is accepted when `shard_epoch` is above + /// the stored one, or equal to it and recorded by this same writer, and refused with + /// [`IndexedStorageError::Fenced`] otherwise. Inserts the record if the key has none. /// /// Monotonic rather than a plain overwrite so that a writer holding a stale epoch cannot walk - /// the record backwards and un-fence itself against the current owner. That holds only for a - /// key that already has a record. A key with none accepts any epoch, whether it was never - /// written, removed by [`Self::delete_oplog_metadata`], or written before the record existed. - /// For such a key the fence cannot tell a stale executor's first open from the owner's; only - /// the lease's admission check bounds that window. + /// the record backwards and un-fence itself against the current owner. Equality is what the + /// writer ([`WriterId`]) settles: a re-open by the process that already holds the epoch is the + /// ordinary case, while another process presenting the same epoch is a manager that lost its + /// state and minted a generation twice - refused here, reported, and minted past. + /// + /// That holds only for a key that already has a record. A key with none accepts any epoch, + /// whether it was never written, removed by [`Self::delete_oplog_metadata`], or written before + /// the record existed. For such a key the fence cannot tell a stale executor's first open from + /// the owner's; only the lease's admission check bounds that window. /// /// The default does nothing and accepts everything: a backend that cannot fence has no record /// to keep. diff --git a/golem-worker-executor/src/storage/indexed/multi_sqlite.rs b/golem-worker-executor/src/storage/indexed/multi_sqlite.rs index 22f916fc0a..4ef31297bf 100644 --- a/golem-worker-executor/src/storage/indexed/multi_sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/multi_sqlite.rs @@ -14,7 +14,7 @@ use super::{ IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, IndexedStorageNamespace, - ScanCursor, ScanResume, + ScanCursor, ScanResume, WriterId, }; use crate::storage::indexed::sqlite::SqliteIndexedStorage; use async_trait::async_trait; @@ -47,6 +47,9 @@ pub struct MultiSqliteIndexedStorage { root_dir: PathBuf, max_connections: u32, foreign_keys: bool, + /// Handed to every per-namespace SQLite storage this opens, so the whole fan-out writes as one + /// process. See [`WriterId`]. + writer_id: WriterId, } struct HashCache { @@ -83,22 +86,33 @@ impl MultiSqliteIndexedStorage { root_dir: root_dir.to_path_buf(), max_connections, foreign_keys, + writer_id: WriterId::process(), } } + /// Writes as `writer_id` rather than as this process's own. The fan-out backend uses it to + /// give every storage it opens one identity, and a test uses it to play two executors racing + /// over one oplog inside a single process. + pub fn for_writer(mut self, writer_id: WriterId) -> Self { + self.writer_id = writer_id; + self + } + async fn init_storage( max_connections: u32, foreign_keys: bool, database: String, + writer_id: WriterId, ) -> Result { let config = DbSqliteConfig { database, max_connections, foreign_keys, }; - SqliteIndexedStorage::configured(&config) + let storage = SqliteIndexedStorage::configured(&config) .await - .map_err(IndexedStorageError::Other) + .map_err(IndexedStorageError::Other)?; + Ok(storage.for_writer(writer_id)) } async fn storage_by_namespace( @@ -187,6 +201,7 @@ impl MultiSqliteIndexedStorage { ) -> Result { let max_connections = self.max_connections; let foreign_keys = self.foreign_keys; + let writer_id = self.writer_id; let db_path = self.root_dir.join(db.clone()).to_string_lossy().to_string(); // Set when this call creates the file, which makes cached listings stale. Checked only on a // cache miss, since a hit means the file is already open. @@ -197,7 +212,7 @@ impl MultiSqliteIndexedStorage { .cache .get_or_insert_simple(&db, async move || { flag.store(!Path::new(&existing).exists(), Ordering::SeqCst); - Self::init_storage(max_connections, foreign_keys, db_path).await + Self::init_storage(max_connections, foreign_keys, db_path, writer_id).await }) .await?; if created.load(Ordering::SeqCst) { diff --git a/golem-worker-executor/src/storage/indexed/postgres.rs b/golem-worker-executor/src/storage/indexed/postgres.rs index 3544fd593b..01c015c639 100644 --- a/golem-worker-executor/src/storage/indexed/postgres.rs +++ b/golem-worker-executor/src/storage/indexed/postgres.rs @@ -14,7 +14,7 @@ use super::{ FencedTxError, IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, - IndexedStorageNamespace, ScanCursor, ScanResume, + IndexedStorageNamespace, ScanCursor, ScanResume, WriterId, }; use crate::services::golem_config::IndexedStoragePostgresConfig; use async_trait::async_trait; @@ -42,6 +42,9 @@ pub struct PostgresIndexedStorage { pool: PostgresPool, drop_prefix_delete_batch_size: u64, semaphore: Option>, + /// Recorded beside the epoch on every oplog this process claims, so an equal epoch from + /// another process is refused rather than shared. One per process; see [`WriterId`]. + writer_id: WriterId, } impl PostgresIndexedStorage { @@ -80,6 +83,7 @@ impl PostgresIndexedStorage { pool, drop_prefix_delete_batch_size: config.drop_prefix_delete_batch_size, semaphore, + writer_id: WriterId::process(), }) } @@ -88,9 +92,18 @@ impl PostgresIndexedStorage { pool, drop_prefix_delete_batch_size: 1024, semaphore: None, + writer_id: WriterId::process(), }) } + /// Writes as `writer_id` rather than as this process's own. The fan-out backend uses it to + /// give every storage it opens one identity, and a test uses it to play two executors racing + /// over one oplog inside a single process. + pub fn for_writer(mut self, writer_id: WriterId) -> Self { + self.writer_id = writer_id; + self + } + pub async fn run_metrics_loop(&self) -> anyhow::Result<()> { self.pool.run_metrics_loop("indexed_storage").await } @@ -136,6 +149,14 @@ impl PostgresIndexedStorage { }) } + /// A stored epoch that will not fit a `u64` is corruption, not a fence. `to_i64` refuses to + /// write one, so a negative column value came from outside this code - and reading it back as + /// `u64` would wrap it into a near-ceiling epoch that fences every writer out of the oplog and, + /// reported as evidence, walks the shard manager's own mint towards its ceiling. + fn epoch_from_i64(value: i64, key: &str) -> String { + format!("Postgres indexed storage read a negative shard epoch {value} for key '{key}'") + } + fn classify_repo_error(err: RepoError, primary_oplog_insert: bool) -> IndexedStorageError { if primary_oplog_insert && err.is_pool_timeout() { IndexedStorageError::Transient(err.to_string()) @@ -396,6 +417,7 @@ impl IndexedStorage for PostgresIndexedStorage { .map_err(|err| Self::classify_repo_error(err, primary_oplog_insert)); } + let writer_id = self.writer_id.to_string(); self.pool .with_tx_err::<(), FencedTxError, _>(svc_name, api_name, |tx| { async move { @@ -404,26 +426,37 @@ impl IndexedStorage for PostgresIndexedStorage { // lost the shard cannot slip a batch in between the check and the insert. // `FOR UPDATE` is what serialises two executors racing over the same oplog. if let Some(expected) = shard_epoch { - let stored: Option<(i64,)> = tx + let stored: Option<(i64, String)> = tx .fetch_optional_as( sqlx::query_as( - "SELECT epoch FROM oplog_metadata WHERE namespace = $1 AND key = $2 FOR UPDATE;", + "SELECT epoch, owner FROM oplog_metadata WHERE namespace = $1 AND key = $2 FOR UPDATE;", ) .bind(namespace.clone()) .bind(key.clone()), ) .await?; - let actual = stored.map(|(epoch,)| ShardEpoch(epoch as u64)); - // Strict equality: the monotonic rule belongs to the upsert. A stored - // epoch above ours means a newer owner has taken over; below ours means - // an open skipped the assertion. Neither is ours to write through. An - // absent row fences too - it is written before the first entry and - // removed before the last. - if actual != Some(expected) { + let mut actual = None; + let mut owner_matches = false; + if let Some((epoch, owner)) = stored { + let epoch = u64::try_from(epoch).map_err(|_| { + FencedTxError::Corrupt(Self::epoch_from_i64(epoch, &key)) + })?; + actual = Some(ShardEpoch(epoch)); + owner_matches = owner == writer_id; + } + // Strict equality on the epoch, and the row's own writer on top of it. A + // stored epoch above ours means a newer owner has taken over; below ours + // means an open skipped the assertion; equal but written by another + // process means a manager that lost its state minted this generation + // twice, and neither of us may write through the other. An absent row + // fences too - it is written before the first entry and removed before + // the last. + if actual != Some(expected) || !owner_matches { return Err(FencedTxError::Fenced { key: key.clone(), expected, actual, + owner_conflict: actual == Some(expected) && !owner_matches, }); } } @@ -459,13 +492,18 @@ impl IndexedStorage for PostgresIndexedStorage { }) } - /// Monotonic compare-and-set on the epoch authorised to write this key. + /// Monotonic compare-and-set on the epoch authorised to write this key, and on the writer + /// holding it. /// /// The `WHERE` on the conflict path is what makes it monotonic: a lower epoch updates no row, /// so while a record exists a writer holding a stale epoch cannot walk it back and un-fence - /// itself against the current owner. With no record there is no conflict and any epoch is - /// inserted. Postgres reports one row affected for an insert and for an accepted update, and - /// zero when the `WHERE` excludes it. + /// itself against the current owner. An equal epoch updates the row only for the process that + /// already recorded it: a re-open by the holder is ordinary, while another process arriving at + /// the same epoch is a shard manager that lost its state and minted the generation twice, and + /// letting it through would put two writers behind one `(shard, epoch)` pair - the thing the + /// epoch exists to tell apart. With no record there is no conflict and any epoch is inserted. + /// Postgres reports one row affected for an insert and for an accepted update, and zero when + /// the `WHERE` excludes it. async fn upsert_oplog_metadata( &self, svc_name: &'static str, @@ -478,37 +516,50 @@ impl IndexedStorage for PostgresIndexedStorage { let namespace = Self::namespace(namespace); let epoch = Self::to_i64(shard_epoch.0, "shard_epoch")?; + let writer_id = self.writer_id.to_string(); + let mut api = self.pool.with_rw(svc_name, api_name); let result = api .execute( sqlx::query( - r#"INSERT INTO oplog_metadata (namespace, key, epoch) VALUES ($1, $2, $3) - ON CONFLICT (namespace, key) DO UPDATE SET epoch = EXCLUDED.epoch - WHERE oplog_metadata.epoch <= EXCLUDED.epoch;"#, + r#"INSERT INTO oplog_metadata (namespace, key, epoch, owner) VALUES ($1, $2, $3, $4) + ON CONFLICT (namespace, key) DO UPDATE SET epoch = EXCLUDED.epoch, owner = EXCLUDED.owner + WHERE oplog_metadata.epoch < EXCLUDED.epoch + OR (oplog_metadata.epoch = EXCLUDED.epoch AND oplog_metadata.owner = EXCLUDED.owner);"#, ) .bind(namespace.clone()) .bind(key) - .bind(epoch), + .bind(epoch) + .bind(writer_id.clone()), ) .await .map_err(Self::classify_repo_error_general)?; if result.rows_affected() == 0 { // Rejected. Read the stored epoch back purely so the error can name it. - let stored: Option<(i64,)> = api + let stored: Option<(i64, String)> = api .fetch_optional_as( sqlx::query_as( - "SELECT epoch FROM oplog_metadata WHERE namespace = $1 AND key = $2;", + "SELECT epoch, owner FROM oplog_metadata WHERE namespace = $1 AND key = $2;", ) .bind(namespace) .bind(key), ) .await .map_err(Self::classify_repo_error_general)?; + let mut actual = None; + let mut owner_matches = false; + if let Some((epoch, owner)) = stored { + let epoch = u64::try_from(epoch) + .map_err(|_| IndexedStorageError::Other(Self::epoch_from_i64(epoch, key)))?; + actual = Some(ShardEpoch(epoch)); + owner_matches = owner == writer_id; + } return Err(IndexedStorageError::Fenced { key: key.to_string(), expected: shard_epoch, - actual: stored.map(|(epoch,)| ShardEpoch(epoch as u64)), + actual, + owner_conflict: actual == Some(shard_epoch) && !owner_matches, }); } diff --git a/golem-worker-executor/src/storage/indexed/sqlite.rs b/golem-worker-executor/src/storage/indexed/sqlite.rs index 70a253856a..89c61bb6c5 100644 --- a/golem-worker-executor/src/storage/indexed/sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/sqlite.rs @@ -14,7 +14,7 @@ use super::{ FencedTxError, IndexedStorage, IndexedStorageError, IndexedStorageMetaNamespace, - IndexedStorageNamespace, ScanCursor, ScanResume, + IndexedStorageNamespace, ScanCursor, ScanResume, WriterId, }; use async_trait::async_trait; use bytes::Bytes; @@ -37,6 +37,9 @@ static DB_MIGRATIONS: include_dir::Dir = include_dir!("$CARGO_MANIFEST_DIR/db/mi #[derive(Debug, Clone)] pub struct SqliteIndexedStorage { pool: SqlitePool, + /// Recorded beside the epoch on every oplog this process claims, so an equal epoch from + /// another process is refused rather than shared. One per process; see [`WriterId`]. + writer_id: WriterId, } impl SqliteIndexedStorage { @@ -55,7 +58,18 @@ impl SqliteIndexedStorage { .await .map_err(|err| format!("Sqlite indexed storage pool initialization failed: {err:?}"))?; - Ok(Self { pool }) + Ok(Self { + pool, + writer_id: WriterId::process(), + }) + } + + /// Writes as `writer_id` rather than as this process's own. The fan-out backend uses it to + /// give every storage it opens one identity, and a test uses it to play two executors racing + /// over one oplog inside a single process. + pub fn for_writer(mut self, writer_id: WriterId) -> Self { + self.writer_id = writer_id; + self } /// Apply the indexed storage migrations on the given sqlite config without @@ -68,7 +82,10 @@ impl SqliteIndexedStorage { } pub fn new(pool: SqlitePool) -> Self { - Self { pool } + Self { + pool, + writer_id: WriterId::process(), + } } fn namespace(namespace: IndexedStorageNamespace) -> String { @@ -117,6 +134,13 @@ impl SqliteIndexedStorage { }) } + /// A stored epoch that will not fit a `u64` is corruption, not a fence: `to_i64` refuses to + /// write one, so a negative column value came from outside this code, and reading it back as + /// `u64` would wrap it into a spuriously huge epoch. + fn epoch_from_i64(value: i64, key: &str) -> String { + format!("SQLite indexed storage read a negative shard epoch {value} for key '{key}'") + } + fn classify_repo_error(err: RepoError) -> IndexedStorageError { if err.is_transient() { IndexedStorageError::Transient(err.to_string()) @@ -336,6 +360,7 @@ impl IndexedStorage for SqliteIndexedStorage { record_db_serialized_size(DB_TYPE, svc_name, entity_name, value.len()); } + let writer_id = self.writer_id.to_string(); self.pool .with_tx_err::<(), FencedTxError, _>(svc_name, api_name, |tx| { Box::pin(async move { @@ -345,21 +370,33 @@ impl IndexedStorage for SqliteIndexedStorage { // check cannot be interleaved. Raising that cap means switching this to // `BEGIN IMMEDIATE`. if let Some(expected) = shard_epoch { - let stored: Option<(i64,)> = tx + let stored: Option<(i64, String)> = tx .fetch_optional_as( sqlx::query_as( - "SELECT epoch FROM oplog_metadata WHERE namespace = ? AND key = ?;", + "SELECT epoch, owner FROM oplog_metadata WHERE namespace = ? AND key = ?;", ) .bind(namespace.clone()) .bind(key.clone()), ) .await?; - let actual = stored.map(|(epoch,)| ShardEpoch(epoch as u64)); - if actual != Some(expected) { + let mut actual = None; + let mut owner_matches = false; + if let Some((epoch, owner)) = stored { + let epoch = u64::try_from(epoch).map_err(|_| { + FencedTxError::Corrupt(Self::epoch_from_i64(epoch, &key)) + })?; + actual = Some(ShardEpoch(epoch)); + owner_matches = owner == writer_id; + } + // The epoch says which generation may write; the writer says which of two + // processes holding that generation recorded it, which only a shard + // manager that lost its state can produce. + if actual != Some(expected) || !owner_matches { return Err(FencedTxError::Fenced { key: key.clone(), expected, actual, + owner_conflict: actual == Some(expected) && !owner_matches, }); } } @@ -390,10 +427,12 @@ impl IndexedStorage for SqliteIndexedStorage { }) } - /// Monotonic compare-and-set: the `WHERE` on the conflict path means a lower epoch updates no - /// row, so while a record exists a stale writer cannot walk it back and un-fence itself. With - /// no record there is no conflict and any epoch is inserted. The unqualified `epoch` there is - /// the existing row's. + /// Monotonic compare-and-set on the epoch and its writer: the `WHERE` on the conflict path + /// means a lower epoch updates no row, so while a record exists a stale writer cannot walk it + /// back and un-fence itself, and an equal epoch updates the row only for the process that + /// recorded it, so two processes cannot share one generation. With no record there is no + /// conflict and any epoch is inserted. The unqualified `epoch`/`owner` there are the existing + /// row's. async fn upsert_oplog_metadata( &self, svc_name: &'static str, @@ -409,36 +448,49 @@ impl IndexedStorage for SqliteIndexedStorage { // rather than `as i64`, which would silently wrap an out-of-range epoch to negative. let epoch = Self::to_i64(shard_epoch.0, "shard_epoch")?; + let writer_id = self.writer_id.to_string(); + let mut api = self.pool.with_rw(svc_name, api_name); let result = api .execute( sqlx::query( - r#"INSERT INTO oplog_metadata (namespace, key, epoch) VALUES (?, ?, ?) - ON CONFLICT(namespace, key) DO UPDATE SET epoch = excluded.epoch - WHERE epoch <= excluded.epoch;"#, + r#"INSERT INTO oplog_metadata (namespace, key, epoch, owner) VALUES (?, ?, ?, ?) + ON CONFLICT(namespace, key) DO UPDATE SET epoch = excluded.epoch, owner = excluded.owner + WHERE epoch < excluded.epoch + OR (epoch = excluded.epoch AND owner = excluded.owner);"#, ) .bind(namespace.clone()) .bind(key) - .bind(epoch), + .bind(epoch) + .bind(writer_id.clone()), ) .await .map_err(Self::classify_repo_error)?; if result.rows_affected() == 0 { - let stored: Option<(i64,)> = api + let stored: Option<(i64, String)> = api .fetch_optional_as( sqlx::query_as( - "SELECT epoch FROM oplog_metadata WHERE namespace = ? AND key = ?;", + "SELECT epoch, owner FROM oplog_metadata WHERE namespace = ? AND key = ?;", ) .bind(namespace) .bind(key), ) .await .map_err(Self::classify_repo_error)?; + let mut actual = None; + let mut owner_matches = false; + if let Some((epoch, owner)) = stored { + let epoch = u64::try_from(epoch) + .map_err(|_| IndexedStorageError::Other(Self::epoch_from_i64(epoch, key)))?; + actual = Some(ShardEpoch(epoch)); + owner_matches = owner == writer_id; + } return Err(IndexedStorageError::Fenced { key: key.to_string(), expected: shard_epoch, - actual: stored.map(|(epoch,)| ShardEpoch(epoch as u64)), + actual, + owner_conflict: actual == Some(shard_epoch) && !owner_matches, }); } diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index 6129ef2c1e..b6f77829a5 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -10986,6 +10986,7 @@ mod tests { agent_id: agent_id.clone(), expected_epoch: ShardEpoch(2), actual_epoch: Some(ShardEpoch(3)), + owner_conflict: false, }; // A latched fence wins over whatever the refusal was flattened into on its way out, and @@ -11845,6 +11846,7 @@ mod tests { agent_id, expected_epoch: golem_common::model::ShardEpoch(3), actual_epoch: Some(golem_common::model::ShardEpoch(4)), + owner_conflict: false, }; // A fence the write path saw in full names both epochs, so an operator reading the log diff --git a/golem-worker-executor/tests/api.rs b/golem-worker-executor/tests/api.rs index 03730cdad5..f9e3e55eba 100644 --- a/golem-worker-executor/tests/api.rs +++ b/golem-worker-executor/tests/api.rs @@ -46,7 +46,7 @@ use golem_test_framework::dsl::{ use golem_worker_executor::services::events::Event; use golem_worker_executor::services::worker_proxy::{WorkerProxy, WorkerProxyError}; use golem_worker_executor::worker::{ - INVOCATION_OWNERSHIP_RECHECK_INTERVAL, WorkerDeletionHook, WorkerDeletionStage, + INVOCATION_OWNERSHIP_RECHECK_INTERVAL, Worker, WorkerDeletionHook, WorkerDeletionStage, }; use golem_worker_executor_test_utils::{ LastUniqueId, PrecompiledComponent, TestContext, TestExecutorOverrides, TestWorkerExecutor, @@ -7512,6 +7512,172 @@ async fn a_stop_through_a_relinquished_generation_leaves_the_next_generation_cac Ok(()) } +/// A retry scheduled before the shard moved must not resume the agent afterwards. +/// +/// A crash schedules the loop's own restart. If the shard is revoked in that window, the agent has +/// been given up, and the loop's backstop (`is_relinquished()` ahead of the retry decision, +/// invocation_loop.rs) has to take the given-up exit instead: no restart here, no `Resumed` written +/// to an oplog the new owner is taking over, and the caller told to reroute. Without the backstop +/// the retry would win the race and resume an agent this executor no longer owns. +#[test] +#[tracing::instrument] +#[timeout(120000)] +async fn a_retry_scheduled_before_a_revoke_does_not_resume_the_agent( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + _tracing: &Tracing, + #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + let component = executor + .component_dep(&context.default_environment_id, host_api_tests) + .store() + .await?; + let agent_id = agent_id!("Clocks", "retry-after-revoke"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &worker_id); + + let invocation = { + let executor = executor.clone(); + let component = component.clone(); + let agent_id = agent_id.clone(); + tokio::spawn( + async move { + executor + .invoke_and_await_agent(&component, &agent_id, "interruption", data_value!()) + .await + } + .in_current_span(), + ) + }; + tokio::time::sleep(Duration::from_secs(5)).await; + + // The crash schedules the loop's restart; the revoke lands while it is pending. + let _ = executor.simulated_crash(&worker_id).await; + revoke_shard_zero(&executor).await?; + + // Given up, so the pending retry must not bring it back: the agent leaves the cache and stays + // out of it. + tokio::time::timeout(Duration::from_secs(30), async { + while executor.worker_is_cached(&owned_agent_id).await { + sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| anyhow!("the agent stayed cached after its shard was revoked"))?; + + let result = tokio::time::timeout(Duration::from_secs(30), invocation) + .await + .map_err(|_| anyhow!("the caller was never answered after the revoke"))??; + assert!( + result.is_err(), + "the invocation must be handed back to the caller to reroute, not completed by an \ + executor that no longer owns the shard" + ); + + sleep(Duration::from_secs(2)).await; + assert!( + !executor.worker_is_cached(&owned_agent_id).await, + "a retry scheduled before the revoke resumed an agent this executor had given up" + ); + + // The shard coming back is what may start it again, from the oplog, as a new generation. + assign_shard_zero(&executor).await?; + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + Ok(()) +} + +/// The other half of the stale-handle contract: a handle kept past its generation must not be able +/// to *start* it either. +/// +/// The stop-side test above pins that a stale handle cannot evict the generation that replaced it. +/// This one pins the guard at the other end (`start_if_needed_internal`, worker/mod.rs): a start +/// through a given-up generation would take permits, could append `Resumed` to an oplog the new +/// owner is now writing, and would publish its failures, by agent id, to the waiters of the +/// generation that replaced it. The caller gets a retriable error instead, and the live generation +/// is untouched. +#[test] +#[tracing::instrument] +async fn a_start_through_a_relinquished_generation_is_refused( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + _tracing: &Tracing, + #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + let component = executor + .component_dep(&context.default_environment_id, host_api_tests) + .store() + .await?; + let agent_id = agent_id!("Clocks", "stale-generation-start"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &worker_id); + + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + let stale = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is not cached after its first invocation"))? + .primary(); + + revoke_shard_zero(&executor).await?; + tokio::time::timeout(Duration::from_secs(10), async { + while executor.worker_is_cached(&owned_agent_id).await { + sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| anyhow!("the agent stayed cached after its shard was revoked"))?; + + assign_shard_zero(&executor).await?; + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + let fresh = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is not cached after the shard came back"))? + .primary(); + assert!( + !Arc::ptr_eq(&stale, &fresh), + "the shard's return must have created a new generation, or this test proves nothing" + ); + + let refused = Worker::start_if_needed(stale.clone()).await; + match refused { + Err(WorkerExecutorError::ShardingNotReady | WorkerExecutorError::OplogFenced { .. }) => {} + Err(other) => panic!( + "a start through a given-up generation must be answered with something the caller can \ + retry on the new owner, got {other}" + ), + Ok(_) => panic!("a start through a relinquished generation was allowed"), + } + + let cached = executor + .active_agent(&owned_agent_id) + .await + .ok_or_else(|| anyhow!("the agent is no longer cached"))? + .primary(); + assert!( + Arc::ptr_eq(&cached, &fresh), + "the cached generation changed under a start through a stale handle" + ); + executor + .invoke_and_await_agent(&component, &agent_id, "sleep_for", data_value!(0.0f64)) + .await?; + Ok(()) +} + /// A caller awaiting an invocation whose oplog write was refused inside a host call must be told /// to reroute. /// diff --git a/golem-worker-executor/tests/hot_update.rs b/golem-worker-executor/tests/hot_update.rs index ffadff5996..676e427484 100644 --- a/golem-worker-executor/tests/hot_update.rs +++ b/golem-worker-executor/tests/hot_update.rs @@ -1588,6 +1588,107 @@ async fn manual_update_on_idle( Ok(()) } +/// A stop arriving while a manual update is in flight must not deadlock either side. +/// +/// This is the shape the final review's F10 was about: the update is enqueued from the invocation +/// loop, and a stop taking the same worker down could wait on the loop that was waiting to enqueue. +/// The enqueue is non-blocking now (`enqueue_update_from_loop`), so both finish. The test is +/// written as a race rather than a fixed order - either outcome is legal, a hang is not - and the +/// timeout is the assertion. +#[test] +#[tracing::instrument] +#[timeout(120000)] +async fn a_stop_racing_a_manual_update_never_deadlocks( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("agent_update_v2")] agent_update_v2: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + + let http_server = TestHttpServer::start().await; + let mut env = HashMap::new(); + env.insert("PORT".to_string(), http_server.port().to_string()); + + let component = executor + .component_dep(&context.default_environment_id, agent_update_v2) + .store() + .await?; + let agent_id = agent_id!("UpdateTest"); + let worker_id = executor + .start_agent_with(&component.id, agent_id.clone(), env, Vec::new()) + .await?; + let mut _log_output_guards = Vec::new(); + _log_output_guards.push(executor.log_output_scoped(&worker_id).await?); + + let updated_component = executor + .update_component(&component.id, "it_agent_update_v3_release") + .await?; + + executor + .invoke_and_await_agent(&component, &agent_id, "f1", data_value!(0u64)) + .await?; + + // Both are issued without awaiting the first: the update goes through the loop-side enqueue + // while the interrupt takes the worker down under it. + let update = { + let executor = executor.clone(); + let worker_id = worker_id.clone(); + let revision = updated_component.revision; + spawn( + async move { + executor + .manual_update_worker(&worker_id, revision, false) + .await + } + .in_current_span(), + ) + }; + let stop = { + let executor = executor.clone(); + let worker_id = worker_id.clone(); + spawn(async move { executor.interrupt(&worker_id).await }.in_current_span()) + }; + + let (update, stop) = tokio::time::timeout(Duration::from_secs(60), async { + tokio::join!(update, stop) + }) + .await + .map_err(|_| { + anyhow::anyhow!("a stop racing a manual update deadlocked: neither call came back") + })?; + // Either may fail on its own terms - the worker is being stopped - but neither may hang, and + // the executor has to stay usable afterwards. + let _ = update?; + let _ = stop?; + + // The worker is still answerable, and on a revision that is one of the two legal outcomes - + // the method set differs between them, so the probe is the metadata rather than a call. + let metadata = tokio::time::timeout( + Duration::from_secs(60), + executor.get_worker_metadata(&worker_id), + ) + .await + .map_err(|_| anyhow::anyhow!("the agent never answered again after the race"))??; + assert!( + metadata.component_revision == updated_component.revision + || metadata.component_revision == ComponentRevision::INITIAL, + "the update either landed or did not, but the revision must be one of the two, got {:?}", + metadata.component_revision + ); + assert!( + !matches!(metadata.status, AgentStatus::Failed), + "a stop racing an update must not fail the agent, got {:?}", + metadata.status + ); + executor.check_oplog_is_queryable(&worker_id).await?; + + drop(executor); + http_server.abort(); + Ok(()) +} + #[test] #[tracing::instrument] async fn manual_update_on_idle_without_save_snapshot( diff --git a/golem-worker-executor/tests/indexed_storage.rs b/golem-worker-executor/tests/indexed_storage.rs index 3e24b54966..a5a3fd0249 100644 --- a/golem-worker-executor/tests/indexed_storage.rs +++ b/golem-worker-executor/tests/indexed_storage.rs @@ -30,7 +30,7 @@ use golem_worker_executor::storage::indexed::redis::RedisIndexedStorage; use golem_worker_executor::storage::indexed::sqlite::SqliteIndexedStorage; use golem_worker_executor::storage::indexed::{ IndexedStorage, IndexedStorageError, IndexedStorageLabelledApi, IndexedStorageMetaNamespace, - IndexedStorageNamespace, ScanCursor, + IndexedStorageNamespace, ScanCursor, WriterId, }; use golem_worker_executor_test_utils::WorkerExecutorTestDependencies; use pretty_assertions::assert_eq; @@ -48,6 +48,16 @@ trait GetIndexedStorage: Debug { /// Whether this backend is expected to enforce the shard-epoch fence. Stated here rather than /// read off the storage so that a backend silently losing its fence fails a test. fn expects_fencing(&self) -> bool; + + /// Two handles onto the SAME store, writing as two different processes - what a shard manager + /// that lost its state can produce by minting one epoch twice. A backend that cannot fence + /// returns two handles that write as the same process; nothing it does is checked. + async fn get_two_writers( + &self, + ) -> ( + Arc, + Arc, + ); } struct InMemoryIndexedStorageWrapper; @@ -68,6 +78,16 @@ impl GetIndexedStorage for InMemoryIndexedStorageWrapper { let kvs = InMemoryIndexedStorage::new(); Arc::new(kvs) } + + async fn get_two_writers( + &self, + ) -> ( + Arc, + Arc, + ) { + let shared: Arc = Arc::new(InMemoryIndexedStorage::new()); + (shared.clone(), shared) + } } #[test_dep(scope = Shared, tagged_as = "in_memory")] @@ -112,6 +132,16 @@ impl GetIndexedStorage for RedisIndexedStorageWrapper { let kvs = RedisIndexedStorage::new(redis_pool); Arc::new(kvs) } + + async fn get_two_writers( + &self, + ) -> ( + Arc, + Arc, + ) { + let shared = self.get_indexed_storage().await; + (shared.clone(), shared) + } } #[test_dep(scope = Shared, tagged_as = "redis")] @@ -165,6 +195,35 @@ impl GetIndexedStorage for SqliteIndexedStorageWrapper { let sis = SqliteIndexedStorage::configured(&config).await.unwrap(); Arc::new(sis) } + + async fn get_two_writers( + &self, + ) -> ( + Arc, + Arc, + ) { + let tempdir = tempfile::tempdir().unwrap(); + let database = tempdir + .path() + .join("indexed.db") + .to_string_lossy() + .into_owned(); + self.tempdirs.lock().unwrap().push(tempdir); + let config = golem_common::config::DbSqliteConfig { + database, + max_connections: 10, + foreign_keys: false, + }; + let first = SqliteIndexedStorage::configured(&config) + .await + .unwrap() + .for_writer(WriterId(Uuid::new_v4())); + let second = SqliteIndexedStorage::configured(&config) + .await + .unwrap() + .for_writer(WriterId(Uuid::new_v4())); + (Arc::new(first), Arc::new(second)) + } } #[test_dep(scope = Shared, tagged_as = "sqlite")] @@ -206,6 +265,22 @@ impl GetIndexedStorage for MultiSqliteIndexedStorageWrapper { let storage = MultiSqliteIndexedStorage::new(&path, 10, true); Arc::new(storage) } + + async fn get_two_writers( + &self, + ) -> ( + Arc, + Arc, + ) { + let tempdir = tempfile::tempdir().unwrap(); + let path = tempdir.path().to_path_buf(); + self.tempdirs.lock().unwrap().push(tempdir); + let first = + MultiSqliteIndexedStorage::new(&path, 10, true).for_writer(WriterId(Uuid::new_v4())); + let second = + MultiSqliteIndexedStorage::new(&path, 10, true).for_writer(WriterId(Uuid::new_v4())); + (Arc::new(first), Arc::new(second)) + } } #[test_dep(scope = Shared, tagged_as = "multi_sqlite")] @@ -225,13 +300,10 @@ impl Debug for PostgresIndexedStorageWrapper { } } -#[async_trait] -impl GetIndexedStorage for PostgresIndexedStorageWrapper { - fn expects_fencing(&self) -> bool { - true - } - - async fn get_indexed_storage(&self) -> Arc { +impl PostgresIndexedStorageWrapper { + /// A fresh database, and the config that reaches it. Separated from `get_indexed_storage` so a + /// second storage can be opened onto the same database as a different writer. + async fn fresh_database(&self) -> IndexedStoragePostgresConfig { let db_name = format!("idx_{}", Uuid::new_v4().simple()); let admin_pool = sqlx::postgres::PgPoolOptions::new() @@ -259,18 +331,46 @@ impl GetIndexedStorage for PostgresIndexedStorageWrapper { acquire_timeout: None, }; - let config = IndexedStoragePostgresConfig { + IndexedStoragePostgresConfig { postgres, drop_prefix_delete_batch_size: 1024, max_concurrent_ops: None, - }; + } + } +} + +#[async_trait] +impl GetIndexedStorage for PostgresIndexedStorageWrapper { + fn expects_fencing(&self) -> bool { + true + } + async fn get_indexed_storage(&self) -> Arc { + let config = self.fresh_database().await; let storage = PostgresIndexedStorage::configured(&config) .await .expect("Cannot create postgres indexed storage"); Arc::new(storage) } + + async fn get_two_writers( + &self, + ) -> ( + Arc, + Arc, + ) { + let config = self.fresh_database().await; + let first = PostgresIndexedStorage::configured(&config) + .await + .expect("Cannot create postgres indexed storage") + .for_writer(WriterId(Uuid::new_v4())); + let second = PostgresIndexedStorage::configured(&config) + .await + .expect("Cannot create postgres indexed storage") + .for_writer(WriterId(Uuid::new_v4())); + (Arc::new(first), Arc::new(second)) + } } #[test_dep(scope = Shared, tagged_as = "postgres")] @@ -1996,6 +2096,188 @@ async fn a_stale_epoch_append_is_refused_and_writes_nothing( } } +#[test] +#[tracing::instrument] +async fn another_writer_at_the_same_epoch_is_refused( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + // A shard manager that lost its state mints from zero again and can hand a live owner's epoch + // to somebody else. The epoch alone cannot separate them, so the row's writer does: the owner + // holds it, and the newcomer is refused at the open rather than sharing the generation. + let fencing = is.expects_fencing(); + let (owner, newcomer) = is.get_two_writers().await; + let key = "fence-two-writers"; + + owner + .upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(4)) + .await + .unwrap(); + + let claim = newcomer + .upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(4)) + .await; + + if !fencing { + claim.unwrap(); + return; + } + + match claim { + Err(IndexedStorageError::Fenced { + expected, + actual, + owner_conflict, + .. + }) => { + assert_eq!(expected, ShardEpoch(4), "expected epoch"); + assert_eq!(actual, Some(ShardEpoch(4)), "stored epoch"); + assert!( + owner_conflict, + "the epochs match, so the refusal has to name the writer as the reason - that is \ + what tells the shard manager to mint past this epoch rather than leave it shared" + ); + } + other => panic!("expected a Fenced error, got {other:?}"), + } + + // And the newcomer cannot write behind the owner's back either. + let append = newcomer + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(1, Bytes::from_static(b"a"))]), + Some(ShardEpoch(4)), + ) + .await; + match append { + Err(IndexedStorageError::Fenced { owner_conflict, .. }) => assert!(owner_conflict), + other => panic!("expected a Fenced error, got {other:?}"), + } + assert_eq!( + owner + .length("svc", "api", ns.ns.clone(), key) + .await + .unwrap(), + 0, + "a refused append writes nothing" + ); + + // The owner is untouched by the attempt. + owner + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(1, Bytes::from_static(b"a"))]), + Some(ShardEpoch(4)), + ) + .await + .unwrap(); +} + +#[test] +#[tracing::instrument] +async fn the_same_writer_re_opens_at_the_same_epoch( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + // The ordinary case the writer column must not break: one process re-opening an oplog it + // already holds, at the epoch it already holds, which happens on every cache eviction. + let is = is.get_indexed_storage().await; + let key = "fence-reopen"; + + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(4)) + .await + .unwrap(); + is.upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(4)) + .await + .unwrap(); + is.append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(1, Bytes::from_static(b"a"))]), + Some(ShardEpoch(4)), + ) + .await + .unwrap(); +} + +#[test] +#[tracing::instrument] +async fn a_newcomer_minted_above_the_collision_takes_the_oplog_over( + deps: &WorkerExecutorTestDependencies, + #[dimension(is)] is: &Arc, + #[tagged_as("ns1")] ns: &IndexedStorageNamespaces, +) { + // The repair the refusal above sets off: the newcomer reports the collision, the shard manager + // mints past it, and the higher epoch takes the oplog over - at which point the old owner is + // the one being refused. + let fencing = is.expects_fencing(); + if !fencing { + return; + } + let (owner, newcomer) = is.get_two_writers().await; + let key = "fence-re-mint"; + + owner + .upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(4)) + .await + .unwrap(); + newcomer + .upsert_oplog_metadata("svc", "api", ns.ns.clone(), key, ShardEpoch(5)) + .await + .unwrap(); + newcomer + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(1, Bytes::from_static(b"a"))]), + Some(ShardEpoch(5)), + ) + .await + .unwrap(); + + let refused = owner + .append_many( + "svc", + "api", + "entity", + &ns.ns, + key, + Arc::from([(2, Bytes::from_static(b"b"))]), + Some(ShardEpoch(4)), + ) + .await; + match refused { + Err(IndexedStorageError::Fenced { + actual, + owner_conflict, + .. + }) => { + assert_eq!(actual, Some(ShardEpoch(5))); + assert!( + !owner_conflict, + "this one is an ordinary takeover, not two writers on one epoch" + ); + } + other => panic!("expected a Fenced error, got {other:?}"), + } +} + #[test] #[tracing::instrument] async fn an_append_ahead_of_the_recorded_epoch_is_refused( diff --git a/golem-worker-service/src/api/invocation_session.rs b/golem-worker-service/src/api/invocation_session.rs index 445d261e7c..fd6267a46b 100644 --- a/golem-worker-service/src/api/invocation_session.rs +++ b/golem-worker-service/src/api/invocation_session.rs @@ -2415,6 +2415,9 @@ fn rejection_code(reason: i32) -> PublicErrorCode { Ok(InvocationRejectionReason::InputConflict) => PublicErrorCode::InputConflict, Ok(InvocationRejectionReason::InputGap) => PublicErrorCode::InputGap, Ok(InvocationRejectionReason::ResourceExhausted) => PublicErrorCode::ResourceExhausted, + // The unary, streaming and agent-RPC paths all reroute on this one; a session client is + // told the same thing so it can reconnect instead of surfacing a server fault. + Ok(InvocationRejectionReason::ShardingNotReady) => PublicErrorCode::ShardingNotReady, _ => PublicErrorCode::InternalError, } } @@ -2446,6 +2449,9 @@ fn safe_rejection_message(code: PublicErrorCode) -> String { PublicErrorCode::ProducerError => "stream producer failed", PublicErrorCode::InvocationFailed => "invocation failed", PublicErrorCode::ProtocolError => "invocation protocol failed", + PublicErrorCode::ShardingNotReady => { + "the agent's shard is moving between executors; retry the invocation" + } PublicErrorCode::InternalError => "invocation failed", } .to_string() From 0b01ed2b0c3087b374198c38e1f20b224f432b3a Mon Sep 17 00:00:00 2001 From: Aditya Salunkhe Date: Fri, 18 Sep 2026 16:12:03 +0530 Subject: [PATCH 6/6] Fix stale and duplicated comments --- golem-shard-manager/src/quota/quota_state.rs | 2 +- golem-shard-manager/src/sharding/model.rs | 7 +++-- .../src/storage/indexed/postgres.rs | 27 +++++++++---------- .../src/storage/indexed/sqlite.rs | 18 ++++++------- golem-worker-executor/src/worker/mod.rs | 5 ++-- 5 files changed, 26 insertions(+), 33 deletions(-) diff --git a/golem-shard-manager/src/quota/quota_state.rs b/golem-shard-manager/src/quota/quota_state.rs index 254c56436f..7ad6409b0c 100644 --- a/golem-shard-manager/src/quota/quota_state.rs +++ b/golem-shard-manager/src/quota/quota_state.rs @@ -30,7 +30,7 @@ use tracing::debug; /// `epoch` here is the caller's claimed epoch, taken straight off the wire (a `renew_lease` / /// `release_lease` argument, itself `golem_common::model::quota::LeaseEpoch(request.epoch)` in /// `grpc.rs` with nothing upstream bounding it) - unlike `pod_lease.epoch`, which only ever -/// advances by exactly one through this state's own `next()` calls. A `u64::MAX` claim can never +/// advances by exactly one through this state's own `checked_next()` calls. A `u64::MAX` claim can never /// legitimately precede a real stored epoch, so it simply fails this check like any other stale /// one, rather than aborting the process. fn precedes(epoch: LeaseEpoch, next: LeaseEpoch) -> bool { diff --git a/golem-shard-manager/src/sharding/model.rs b/golem-shard-manager/src/sharding/model.rs index 607dc4febd..4aae75a499 100644 --- a/golem-shard-manager/src/sharding/model.rs +++ b/golem-shard-manager/src/sharding/model.rs @@ -611,10 +611,9 @@ impl ShardLeaseState { // but a claim of `u64::MAX - 1` does not, and minting past it succeeds, landing // exactly on `u64::MAX`. Either way the wire carries a raw `u64` with nothing // upstream bounding it, and a `u64::MAX` epoch must never reach `shard_epochs`: the - // next ordinary reassignment mints past whatever is stored there - // (`next_epoch_for`), and `ShardEpoch::next` panics on it - aborting this process, - // and again on every retry of the same report. So a candidate that would land on it - // is dropped here, the same stance as the out-of-range shard id above. + // next ordinary reassignment has nothing left to mint past it (`next_epoch_for`), so + // the shard could never change owner again. A candidate that would land on it is + // dropped here, the same stance as the out-of-range shard id above. let candidate = if mints_past_claim { claimed_epoch.checked_next() } else { diff --git a/golem-worker-executor/src/storage/indexed/postgres.rs b/golem-worker-executor/src/storage/indexed/postgres.rs index 01c015c639..d71fcb1615 100644 --- a/golem-worker-executor/src/storage/indexed/postgres.rs +++ b/golem-worker-executor/src/storage/indexed/postgres.rs @@ -153,7 +153,7 @@ impl PostgresIndexedStorage { /// write one, so a negative column value came from outside this code - and reading it back as /// `u64` would wrap it into a near-ceiling epoch that fences every writer out of the oplog and, /// reported as evidence, walks the shard manager's own mint towards its ceiling. - fn epoch_from_i64(value: i64, key: &str) -> String { + fn negative_epoch_message(value: i64, key: &str) -> String { format!("Postgres indexed storage read a negative shard epoch {value} for key '{key}'") } @@ -439,7 +439,7 @@ impl IndexedStorage for PostgresIndexedStorage { let mut owner_matches = false; if let Some((epoch, owner)) = stored { let epoch = u64::try_from(epoch).map_err(|_| { - FencedTxError::Corrupt(Self::epoch_from_i64(epoch, &key)) + FencedTxError::Corrupt(Self::negative_epoch_message(epoch, &key)) })?; actual = Some(ShardEpoch(epoch)); owner_matches = owner == writer_id; @@ -492,18 +492,14 @@ impl IndexedStorage for PostgresIndexedStorage { }) } - /// Monotonic compare-and-set on the epoch authorised to write this key, and on the writer - /// holding it. + /// Postgres's half of [`IndexedStorage::upsert_oplog_metadata`], which states the rule this + /// enforces. /// - /// The `WHERE` on the conflict path is what makes it monotonic: a lower epoch updates no row, - /// so while a record exists a writer holding a stale epoch cannot walk it back and un-fence - /// itself against the current owner. An equal epoch updates the row only for the process that - /// already recorded it: a re-open by the holder is ordinary, while another process arriving at - /// the same epoch is a shard manager that lost its state and minted the generation twice, and - /// letting it through would put two writers behind one `(shard, epoch)` pair - the thing the - /// epoch exists to tell apart. With no record there is no conflict and any epoch is inserted. - /// Postgres reports one row affected for an insert and for an accepted update, and zero when - /// the `WHERE` excludes it. + /// The `WHERE` on the conflict path is where it lives: `epoch < EXCLUDED.epoch` for a higher + /// generation, or `= EXCLUDED.epoch AND owner = EXCLUDED.owner` for the same process re-opening + /// at the one it holds. With no record there is no conflict and any epoch is inserted. Postgres + /// reports one row affected for an insert and for an accepted update, and zero when the `WHERE` + /// excludes it - which is what the read-back below turns into a fence. async fn upsert_oplog_metadata( &self, svc_name: &'static str, @@ -550,8 +546,9 @@ impl IndexedStorage for PostgresIndexedStorage { let mut actual = None; let mut owner_matches = false; if let Some((epoch, owner)) = stored { - let epoch = u64::try_from(epoch) - .map_err(|_| IndexedStorageError::Other(Self::epoch_from_i64(epoch, key)))?; + let epoch = u64::try_from(epoch).map_err(|_| { + IndexedStorageError::Other(Self::negative_epoch_message(epoch, key)) + })?; actual = Some(ShardEpoch(epoch)); owner_matches = owner == writer_id; } diff --git a/golem-worker-executor/src/storage/indexed/sqlite.rs b/golem-worker-executor/src/storage/indexed/sqlite.rs index 89c61bb6c5..bb1dc527a8 100644 --- a/golem-worker-executor/src/storage/indexed/sqlite.rs +++ b/golem-worker-executor/src/storage/indexed/sqlite.rs @@ -137,7 +137,7 @@ impl SqliteIndexedStorage { /// A stored epoch that will not fit a `u64` is corruption, not a fence: `to_i64` refuses to /// write one, so a negative column value came from outside this code, and reading it back as /// `u64` would wrap it into a spuriously huge epoch. - fn epoch_from_i64(value: i64, key: &str) -> String { + fn negative_epoch_message(value: i64, key: &str) -> String { format!("SQLite indexed storage read a negative shard epoch {value} for key '{key}'") } @@ -383,7 +383,7 @@ impl IndexedStorage for SqliteIndexedStorage { let mut owner_matches = false; if let Some((epoch, owner)) = stored { let epoch = u64::try_from(epoch).map_err(|_| { - FencedTxError::Corrupt(Self::epoch_from_i64(epoch, &key)) + FencedTxError::Corrupt(Self::negative_epoch_message(epoch, &key)) })?; actual = Some(ShardEpoch(epoch)); owner_matches = owner == writer_id; @@ -427,12 +427,9 @@ impl IndexedStorage for SqliteIndexedStorage { }) } - /// Monotonic compare-and-set on the epoch and its writer: the `WHERE` on the conflict path - /// means a lower epoch updates no row, so while a record exists a stale writer cannot walk it - /// back and un-fence itself, and an equal epoch updates the row only for the process that - /// recorded it, so two processes cannot share one generation. With no record there is no - /// conflict and any epoch is inserted. The unqualified `epoch`/`owner` there are the existing - /// row's. + /// SQLite's half of [`IndexedStorage::upsert_oplog_metadata`], which states the rule this + /// enforces. The unqualified `epoch`/`owner` in the `WHERE` are the existing row's, and + /// `excluded` is the row being written. async fn upsert_oplog_metadata( &self, svc_name: &'static str, @@ -481,8 +478,9 @@ impl IndexedStorage for SqliteIndexedStorage { let mut actual = None; let mut owner_matches = false; if let Some((epoch, owner)) = stored { - let epoch = u64::try_from(epoch) - .map_err(|_| IndexedStorageError::Other(Self::epoch_from_i64(epoch, key)))?; + let epoch = u64::try_from(epoch).map_err(|_| { + IndexedStorageError::Other(Self::negative_epoch_message(epoch, key)) + })?; actual = Some(ShardEpoch(epoch)); owner_matches = owner == writer_id; } diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index b6f77829a5..6968457f90 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -1163,9 +1163,8 @@ impl Worker { .await; } - /// What anyone waiting on this agent is told. Every variant is one the worker service answers - /// by refreshing its routing table and retrying, so the invocation lands on the new owner - /// rather than failing. + /// [`RelinquishReason::to_error`] for the reason recorded for this agent, or + /// `ShardingNotReady` when none is recorded yet. pub(crate) fn relinquish_error(&self) -> WorkerExecutorError { self.relinquishment .get()