From 2736fe9a1d0dbc03927ab4ebfdea9e5429ccbd58 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Fri, 13 Nov 2026 07:51:34 -0800 Subject: [PATCH 1/2] Read the leader and candidate leader terms as one pair When log replay finishes, CcNode::FinishLogGroupReplay() stores the node group's leader term and then clears its candidate leader term. Code that read the leader term first and the candidate leader term second could load the leader term before the first store and the candidate leader term after the second, and conclude that the node was neither leader nor candidate. SplitFlushRangeOp::Forward() took that for an invalid recovered term and dropped the recovered range split without a log line. Its prepare log and its range write locks stayed behind with no owner: a new split of the range blocked at its first lock, orphan lock recovery looped because the log service kept answering "committed", and the node did not shut down. Sharder::LeaderTerms() now returns both terms, loading the candidate leader term first, the reverse of the store order, so the pair never shows both unset across the hand-over. Every site that read the leader term and then the candidate leader term goes through it: the split and data migration operations, TransactionExecution::CheckLeaderTerm(), CcNode::PinData(), Sharder::FetchLeaderTerm(), the catalog read of a recovering request, the "wait for replay" loops and the remaining checks. Each site keeps its comparisons. RangeBucketCcMap's read used one term per path and only drops its unused leader term load. The two operations now log when they drop a recovered transaction. Co-Authored-By: Claude Fable 5.2 --- docs/06-distribution-and-clustering.md | 11 +++- tx_service/include/cc/catalog_cc_map.h | 19 +++++-- tx_service/include/cc/range_bucket_cc_map.h | 5 +- tx_service/include/sharder.h | 61 +++++++++++++++++++++ tx_service/include/tx_execution.h | 13 +++-- tx_service/src/cc/cc_req_misc.cpp | 8 +-- tx_service/src/cc/local_cc_shards.cpp | 5 +- tx_service/src/checkpointer.cpp | 4 +- tx_service/src/fault/cc_node.cpp | 34 +++++------- tx_service/src/remote/cc_node_service.cpp | 12 ++-- tx_service/src/sharder.cpp | 22 +++++++- tx_service/src/tx_index_operation.cpp | 3 +- tx_service/src/tx_operation.cpp | 36 +++++++++--- 13 files changed, 170 insertions(+), 63 deletions(-) diff --git a/docs/06-distribution-and-clustering.md b/docs/06-distribution-and-clustering.md index 38fc7edfc..6efc0c25f 100644 --- a/docs/06-distribution-and-clustering.md +++ b/docs/06-distribution-and-clustering.md @@ -83,9 +83,13 @@ NG's only replay request to the log groups every node the NG, its new leader and the term (`NotifyNewLeaderStart`, best effort). The leader term is published only after every log group has completed replay, so normal CC -requests cannot observe a partially recovered in-memory replica. A synchronized -standby may retain its warm state and replay only the missing tail; a cold -candidate rebuilds from storage and WAL. +requests cannot observe a partially recovered in-memory replica. The candidate +term is cleared only after the leader term is published, so from election to +stepdown at least one of the two is set. They are separate values; code that +needs both reads them as one pair (`Sharder::LeaderTerms()`), candidate term +first, and so never finds both unset while the node passes from candidate to +leader. A synchronized standby may retain its warm state and replay only the +missing tail; a cold candidate rebuilds from storage and WAL. On stepdown, the serving and candidate terms are invalidated before NG-owned memory is released. Code that accesses such memory outside the shard lifecycle @@ -168,6 +172,7 @@ candidate becomes a complete in-memory replica. | Cluster topology, placement functions, leader caches, terms, and data pinning are owned by `Sharder` | `tx_service/include/sharder.h`; `tx_service/src/sharder.cpp` | | Election and membership cross the host-manager RPC boundary | `tx_service/include/proto/cc_request.proto`; `tx_service/src/remote/cc_node_service.cpp`; `tx_service/src/sharder.cpp` | | An NG becomes serviceable only after leader recovery completes | `tx_service/include/fault/cc_node.h`; `tx_service/src/fault/cc_node.cpp`; `tx_service/src/fault/log_replay_service.cpp` | +| The leader term and the candidate term are read as one pair, candidate term first | `tx_service/include/sharder.h` (`NodeGroupTerms`, `Sharder::LeaderTerms`); `tx_service/src/fault/cc_node.cpp` (`FinishLogGroupReplay`) | | Local and remote requests converge on the CC-request/shard execution model | `tx_service/include/remote/remote_cc_handler.h`; `tx_service/include/remote/remote_cc_request.h`; `tx_service/src/remote/cc_stream_sender.cpp`; `tx_service/src/remote/cc_stream_receiver.cpp` | | Cluster configuration and bucket ownership are transactional metadata | `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/include/cluster_config_record.h`; `tx_service/include/cc/range_bucket_cc_map.h`; `tx_service/src/cc/local_cc_shards.cpp` | | Scaling and data migration are resumable transaction operations | `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp`; `tx_service/tx-log-protos/log.proto` | diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 2ce22163a..cea7f6d09 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -322,14 +322,21 @@ class CatalogCcMap assert(req.IsLocal()); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); - + int64_t ng_term = -1; if (req.IsInRecovering()) { - ng_term = ng_term > 0 - ? ng_term - : Sharder::Instance().CandidateLeaderTerm(ng_id); + // A recovering request is also served in the candidate leader + // term. The leader term and the candidate leader term come from + // one read, so that neither is missed when replay finishes. + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id); + ng_term = + std::max(terms.leader_, Sharder::Instance().StandbyNodeTerm()); + ng_term = ng_term > 0 ? ng_term : terms.candidate_; + } + else + { + ng_term = std::max(Sharder::Instance().LeaderTerm(ng_id), + Sharder::Instance().StandbyNodeTerm()); } CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_ReadCc", { diff --git a/tx_service/include/cc/range_bucket_cc_map.h b/tx_service/include/cc/range_bucket_cc_map.h index 6805aa5fd..a2ecff21a 100644 --- a/tx_service/include/cc/range_bucket_cc_map.h +++ b/tx_service/include/cc/range_bucket_cc_map.h @@ -90,14 +90,15 @@ class RangeBucketCcMap assert(req.IsLocal()); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); + int64_t ng_term = -1; if (req.IsInRecovering()) { ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id); } else { - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); + ng_term = std::max(Sharder::Instance().LeaderTerm(ng_id), + Sharder::Instance().StandbyNodeTerm()); } if (ng_term < 0) diff --git a/tx_service/include/sharder.h b/tx_service/include/sharder.h index eccbb1d07..2fcfe3e9d 100644 --- a/tx_service/include/sharder.h +++ b/tx_service/include/sharder.h @@ -157,6 +157,47 @@ struct ClusterConfig std::shared_ptr> nodes_configs_; }; +/** + * @brief The leader term and the candidate leader term of a cc node group on + * this node, read together by Sharder::LeaderTerms(). + */ +struct NodeGroupTerms +{ + /** + * @brief Whether this node is replaying the node group's log and does not + * lead the group yet. + * + * @return true if the candidate leader term is set and the leader term is + * not. + */ + bool Replaying() const + { + return leader_ < 0 && candidate_ > 0; + } + + /** + * @brief The term in which this node leads the node group or is about to. + * + * @return The leader term if it is positive, else the candidate leader + * term, which is negative when this node is neither. + */ + int64_t LeaderOrCandidate() const + { + return leader_ > 0 ? leader_ : candidate_; + } + + /** + * The term in which this node leads the node group; negative if it does + * not. + */ + int64_t leader_{-1}; + /** + * The term in which this node is replaying the node group's log before it + * leads the group; negative if it is not. + */ + int64_t candidate_{-1}; +}; + /** * Sharder is the collection of services supplied by TxService, which includes: * 1. cc_stream rpc service, which transfers CcRequests between different @@ -348,6 +389,26 @@ class Sharder int64_t CandidateLeaderTerm(uint32_t ng_id) const; + /** + * @brief Reads the leader term and the candidate leader term of a cc node + * group on this node as one consistent pair. + * + * The two terms are separate atomics. When log replay finishes, the cc + * node publishes the leader term and only then clears the candidate + * leader term (CcNode::FinishLogGroupReplay()). This function loads the + * candidate leader term first and the leader term second, the reverse of + * that order, so the pair it returns never shows both terms unset while + * the node passes from candidate leader to leader. Code that looks at + * both terms must call this function: LeaderTerm() followed by + * CandidateLeaderTerm() can see the old leader term and the new candidate + * leader term, and conclude that the node is neither. + * + * @param ng_id The cc node group ID. + * @return The two terms; both negative before the cc nodes are + * initialized. + */ + NodeGroupTerms LeaderTerms(uint32_t ng_id) const; + /** * @brief Updates the leader cache of all cc node groups. * diff --git a/tx_service/include/tx_execution.h b/tx_service/include/tx_execution.h index c70873172..70ced1c9e 100644 --- a/tx_service/include/tx_execution.h +++ b/tx_service/include/tx_execution.h @@ -276,14 +276,17 @@ class TransactionExecution : public LinkedTransaction bool CheckLeaderTerm() const { NodeGroupId ng_id = TxCcNodeId(); - if (Sharder::Instance().CheckLeaderTerm(ng_id, TxTerm()) || - (TxStatus() == TxnStatus::Recovering && - Sharder::Instance().CandidateLeaderTerm(ng_id) >= 0)) + if (TxStatus() != TxnStatus::Recovering) { - return true; + return Sharder::Instance().CheckLeaderTerm(ng_id, TxTerm()); } - return false; + // A recovered transaction is also valid while the node is still the + // candidate leader. Both terms come from one read, so that the check + // does not fail at the moment replay finishes. + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id); + return (terms.leader_ >= 0 && terms.leader_ == TxTerm()) || + terms.candidate_ >= 0; } bool CheckStandbyTerm() const diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 3ab95960f..bc7919db2 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -120,11 +120,9 @@ bool FetchCatalogCc::ValidTermCheck() } else { - int64_t cc_ng_term = Sharder::Instance().LeaderTerm(cc_ng_id_); - if (cc_ng_term < 0) - { - cc_ng_term = Sharder::Instance().CandidateLeaderTerm(cc_ng_id_); - } + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(cc_ng_id_); + const int64_t cc_ng_term = + terms.leader_ < 0 ? terms.candidate_ : terms.leader_; if (cc_ng_term != cc_ng_term_) { diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 341122087..fce7992d9 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -1748,8 +1748,9 @@ uint64_t LocalCcShards::CountSlices(const TableName &table_name, TableName range_table_name(table_name.StringView(), TableType::RangePartition, table_name.Engine()); - assert(Sharder::Instance().LeaderTerm(local_ng_id) >= 0 || - Sharder::Instance().CandidateLeaderTerm(local_ng_id) >= 0); + [[maybe_unused]] const NodeGroupTerms local_ng_terms = + Sharder::Instance().LeaderTerms(local_ng_id); + assert(local_ng_terms.leader_ >= 0 || local_ng_terms.candidate_ >= 0); uint64_t slices = 0; diff --git a/tx_service/src/checkpointer.cpp b/tx_service/src/checkpointer.cpp index d273a961d..a3fef20d9 100644 --- a/tx_service/src/checkpointer.cpp +++ b/tx_service/src/checkpointer.cpp @@ -164,8 +164,8 @@ bool Checkpointer::IsCurrentCheckpointTerm(NodeGroupId node_group_id, } Sharder &sharder = Sharder::Instance(); - if (sharder.LeaderTerm(node_group_id) == term || - sharder.CandidateLeaderTerm(node_group_id) == term) + const NodeGroupTerms terms = sharder.LeaderTerms(node_group_id); + if (terms.leader_ == term || terms.candidate_ == term) { return true; } diff --git a/tx_service/src/fault/cc_node.cpp b/tx_service/src/fault/cc_node.cpp index 7ec610ccc..c91a08729 100644 --- a/tx_service/src/fault/cc_node.cpp +++ b/tx_service/src/fault/cc_node.cpp @@ -61,13 +61,12 @@ CcNode::CcNode(const uint32_t ng_id, bool CcNode::CheckLogGroupReplayFinished(uint32_t log_group_id, int64_t ng_term) { std::lock_guard lk(recovery_mux_); - int64_t term = Sharder::Instance().LeaderTerm(ng_id_); - if (term >= ng_term) + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id_); + if (terms.leader_ >= ng_term) { return true; } - int64_t candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id_); - if (candidate_term > ng_term) + if (terms.candidate_ > ng_term) { return true; } @@ -122,6 +121,9 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id, recovered_log_groups_.clear(); candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id_); + // The leader term is set before the candidate leader term is cleared. + // Sharder::LeaderTerms() reads the two in the reverse order and relies + // on this one to never see both unset during the hand-over. Sharder::Instance().SetLeaderTerm(ng_id_, candidate_term); LOG(INFO) << "The leader of cc node group ng#" << ng_id_ << " with the term " << candidate_term @@ -135,19 +137,16 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id, int64_t CcNode::PinData() { std::unique_lock lk(pinning_threads_mux_); - int64_t leader_term = Sharder::Instance().LeaderTerm(ng_id_); + // The leader term, or the candidate leader term while the node is + // replaying. The two are read as one pair: read one after the other, both + // can be missed at the moment replay finishes, and the caller would be + // told that this node holds no term. + const int64_t leader_term = + Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate(); if (leader_term > 0) { pinning_threads_++; } - else - { - leader_term = Sharder::Instance().CandidateLeaderTerm(ng_id_); - if (leader_term > 0) - { - pinning_threads_++; - } - } return leader_term; } @@ -979,8 +978,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, while (cntl.Failed() || start_follow_resp.error()) { - if (Sharder::Instance().LeaderTerm(ng_id_) > 0 || - Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate() > 0 || Sharder::Instance().StandbyNodeTerm() > 0 || Sharder::Instance().CandidateStandbyNodeTerm() > 0) { @@ -1044,8 +1042,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, int64_t standby_term = (primary_term << 32) | start_follow_resp.subscribe_id(); // verify we're not on a newer term. - if (Sharder::Instance().LeaderTerm(ng_id_) > 0 || - Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate() > 0 || Sharder::Instance().StandbyNodeTerm() > standby_term || Sharder::Instance().CandidateStandbyNodeTerm() > standby_term) { @@ -1262,8 +1259,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, } // We only need to retry if the message is not delivered. - if (Sharder::Instance().LeaderTerm(ng_id_) > 0 || - Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate() > 0 || (Sharder::Instance().StandbyNodeTerm() != standby_term && Sharder::Instance().CandidateStandbyNodeTerm() != standby_term)) { diff --git a/tx_service/src/remote/cc_node_service.cpp b/tx_service/src/remote/cc_node_service.cpp index 104e36b4f..da190e24d 100644 --- a/tx_service/src/remote/cc_node_service.cpp +++ b/tx_service/src/remote/cc_node_service.cpp @@ -776,8 +776,7 @@ void CcNodeService::FlushDataAll(::google::protobuf::RpcController *controller, &local_shards = this->local_shards_]() { int64_t leader_term = INIT_TERM; - while (Sharder::Instance().LeaderTerm(ng_id) < 0 && - Sharder::Instance().CandidateLeaderTerm(ng_id) > 0) + while (Sharder::Instance().LeaderTerms(ng_id).Replaying()) { // The RPC server can receive the remote request, but this // node has not finish log replay, including data(.pk) log and @@ -1172,8 +1171,7 @@ void CcNodeService::GenerateSkFromPk( tx_number, tx_term]() { - while (Sharder::Instance().LeaderTerm(ng_id) < 0 && - Sharder::Instance().CandidateLeaderTerm(ng_id) > 0) + while (Sharder::Instance().LeaderTerms(ng_id).Replaying()) { // The RPC server can receive the remote request, but this // node has not finish log replay, including data(.pk) log and @@ -2183,9 +2181,9 @@ void CcNodeService::FetchNodeInfo( response->set_ng_id(ng_id); response->set_node_id(node_id); - const int64_t leader_term = Sharder::Instance().LeaderTerm(ng_id); - const int64_t candidate_term = - Sharder::Instance().CandidateLeaderTerm(ng_id); + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id); + const int64_t leader_term = terms.leader_; + const int64_t candidate_term = terms.candidate_; if (leader_term > 0) { response->set_role(NodeRole::LeaderNode); diff --git a/tx_service/src/sharder.cpp b/tx_service/src/sharder.cpp index aa29bc9bd..ee13a6f6b 100644 --- a/tx_service/src/sharder.cpp +++ b/tx_service/src/sharder.cpp @@ -636,6 +636,25 @@ int64_t Sharder::CandidateLeaderTerm(uint32_t ng_id) const return candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire); } +NodeGroupTerms Sharder::LeaderTerms(uint32_t ng_id) const +{ + NodeGroupTerms terms; + if (!cc_nodes_init_.load(std::memory_order_acquire)) + { + return terms; + } + + // The candidate leader term first: when replay finishes, the cc node + // stores the leader term and then clears the candidate leader term. A + // load that sees the candidate leader term cleared is therefore followed + // by a load that sees the leader term set. In the opposite order, both + // loads can miss the term the node holds. + terms.candidate_ = + candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire); + terms.leader_ = leader_term_cache_[ng_id].load(std::memory_order_acquire); + return terms; +} + void Sharder::UpdateLeaders() { std::shared_lock cnf_lk(cluster_cnf_mux_); @@ -682,8 +701,7 @@ int64_t Sharder::FetchLeaderTerm(uint32_t ng_id, uint32_t node_id) { if (node_id == node_id_) { - const int64_t leader_term = LeaderTerm(ng_id); - return leader_term > 0 ? leader_term : CandidateLeaderTerm(ng_id); + return LeaderTerms(ng_id).LeaderOrCandidate(); } std::shared_ptr channel = GetCcNodeServiceChannel(node_id); if (channel == nullptr) diff --git a/tx_service/src/tx_index_operation.cpp b/tx_service/src/tx_index_operation.cpp index 0af3a01f2..f0a1f130a 100644 --- a/tx_service/src/tx_index_operation.cpp +++ b/tx_service/src/tx_index_operation.cpp @@ -2473,8 +2473,7 @@ void UpsertTableIndexOp::HandleRangeTask( &task_value, &dispatch_func]() { - while (Sharder::Instance().LeaderTerm(range_owner) < 0 && - Sharder::Instance().CandidateLeaderTerm(range_owner) > 0) + while (Sharder::Instance().LeaderTerms(range_owner).Replaying()) { // Waiting until log replay finished on this node // group. Including data(.pk) log and and diff --git a/tx_service/src/tx_operation.cpp b/tx_service/src/tx_operation.cpp index 3c97e10db..850ed24f7 100644 --- a/tx_service/src/tx_operation.cpp +++ b/tx_service/src/tx_operation.cpp @@ -3349,16 +3349,26 @@ void SplitFlushRangeOp::ClearInfos() void SplitFlushRangeOp::Forward(TransactionExecution *txm) { - if (txm->TxStatus() == TxnStatus::Recovering && - Sharder::Instance().LeaderTerm(txm->TxCcNodeId()) < 0) + // The leader term and the candidate leader term, read as one pair. Read + // one after the other, both can be missed at the moment replay finishes, + // and the recovered split would be dropped although its term is still + // valid, leaving its prepare log and its range locks without an owner. + const NodeGroupTerms terms = + Sharder::Instance().LeaderTerms(txm->TxCcNodeId()); + if (txm->TxStatus() == TxnStatus::Recovering && terms.leader_ < 0) { // This is a recovered tx and replay is not done yet. We should wait for // replay finish before forwarding tx machine. - if (Sharder::Instance().CandidateLeaderTerm(txm->TxCcNodeId()) != - txm->TxTerm()) + if (terms.candidate_ != txm->TxTerm()) { // Recovered term is invalid. Do not call ForceToFinish as it will // cause infinite recursive call. Clean up tx state directly. + LOG(WARNING) << "Split Flush transaction recovered in term " + << txm->TxTerm() + << " is dropped, txn: " << txm->TxNumber() + << ", node group #" << txm->TxCcNodeId() + << " leader term " << terms.leader_ + << ", candidate leader term " << terms.candidate_; txm->bool_resp_->Finish(false); ClearInfos(); @@ -6415,17 +6425,27 @@ void DataMigrationOp::PrepareNextRoundBuckets() void DataMigrationOp::Forward(TransactionExecution *txm) { - if (txm->TxStatus() == TxnStatus::Recovering && - Sharder::Instance().LeaderTerm(txm->TxCcNodeId()) < 0) + // The leader term and the candidate leader term, read as one pair. Read + // one after the other, both can be missed at the moment replay finishes, + // and the recovered migration would be dropped although its term is still + // valid. + const NodeGroupTerms terms = + Sharder::Instance().LeaderTerms(txm->TxCcNodeId()); + if (txm->TxStatus() == TxnStatus::Recovering && terms.leader_ < 0) { // This is a recovered tx and replay is not done yet. We should wait // for replay finish before forwarding tx machine. - if (Sharder::Instance().CandidateLeaderTerm(txm->TxCcNodeId()) != - txm->TxTerm()) + if (terms.candidate_ != txm->TxTerm()) { // Recovered term is invalid. Do not call ForceToFinish as it // will cause infinite recursive call. Clean up tx state // directly. + LOG(WARNING) << "Data migration transaction recovered in term " + << txm->TxTerm() + << " is dropped, txn: " << txm->TxNumber() + << ", node group #" << txm->TxCcNodeId() + << " leader term " << terms.leader_ + << ", candidate leader term " << terms.candidate_; Clear(); txm->state_stack_.pop_back(); assert(txm->state_stack_.empty()); From 29544ecbcf5cb6a1224651ba25fd67b0e9053ee7 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 16 Nov 2026 00:59:15 -0800 Subject: [PATCH 2/2] Accept a recovered transaction only in its own term; test the term reads TransactionExecution::CheckLeaderTerm() accepted a recovered transaction whenever this node had a candidate leader term. If the node loses the node group and is elected again before a recovered transaction finishes, the replay in the new term recovers the transaction again, while the one recovered in the old term still passed the check until that replay finished. A recovered transaction now passes only while the candidate leader term equals its own term. The check keeps its original shape and reads both terms through one LeaderTerms() call. Sharder::LeaderTerms() moves into the header, with an overload that calls a function between its two loads, so that a test can promote the candidate leader at the one point where the load order matters. NodeGroupTerms::LeaderOrCandidate() is renamed LeadershipTerm(). The comments on the two term caches, LeaderTerms() and CcNode::PinData() now say plainly what each holds or returns. LeaderTerms-Test checks both changes: LeaderTerms() finds the term whether the promotion lands before, between or after its two loads, and CheckLeaderTerm() accepts a recovered transaction only while this node leads the group or replays its log in the transaction's term. Co-Authored-By: Claude Opus 5.5 --- tx_service/include/sharder.h | 81 ++++++++- tx_service/include/tx_execution.h | 27 +-- tx_service/src/fault/cc_node.cpp | 17 +- tx_service/src/sharder.cpp | 21 +-- tx_service/tests/CMakeLists.txt | 1 + tx_service/tests/LeaderTerms-Test.cpp | 235 ++++++++++++++++++++++++++ 6 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 tx_service/tests/LeaderTerms-Test.cpp diff --git a/tx_service/include/sharder.h b/tx_service/include/sharder.h index 2fcfe3e9d..c925bc052 100644 --- a/tx_service/include/sharder.h +++ b/tx_service/include/sharder.h @@ -176,12 +176,19 @@ struct NodeGroupTerms } /** - * @brief The term in which this node leads the node group or is about to. + * @brief The leader term if this node leads the node group; otherwise the + * candidate leader term if this node is still replaying the group's log. + * + * The candidate leader term and the leader term are one term at two + * stages: a node elected in term T publishes T as its candidate leader + * term while it replays the group's log, and publishes the same T as its + * leader term when replay finishes (CcNode::FinishLogGroupReplay()). * * @return The leader term if it is positive, else the candidate leader - * term, which is negative when this node is neither. + * term; negative if this node neither leads the group nor replays its + * log. */ - int64_t LeaderOrCandidate() const + int64_t LeadershipTerm() const { return leader_ > 0 ? leader_ : candidate_; } @@ -400,14 +407,54 @@ class Sharder * that order, so the pair it returns never shows both terms unset while * the node passes from candidate leader to leader. Code that looks at * both terms must call this function: LeaderTerm() followed by - * CandidateLeaderTerm() can see the old leader term and the new candidate - * leader term, and conclude that the node is neither. + * CandidateLeaderTerm() can load the leader term before it is set and + * the candidate leader term after it is cleared, and conclude that the + * node is neither leader nor candidate leader. * * @param ng_id The cc node group ID. * @return The two terms; both negative before the cc nodes are * initialized. */ - NodeGroupTerms LeaderTerms(uint32_t ng_id) const; + NodeGroupTerms LeaderTerms(uint32_t ng_id) const + { + return LeaderTerms(ng_id, [] {}); + } + + /** + * @brief LeaderTerms(ng_id), calling a function between the two loads. + * + * For tests: the function can change the terms at the one point where + * the order of the loads matters, for example promote the candidate + * leader the way CcNode::FinishLogGroupReplay() does. + * + * @param ng_id The cc node group ID. + * @param between_loads Called after the candidate leader term is loaded + * and before the leader term is loaded. + * @return The two terms; both negative before the cc nodes are + * initialized, in which case between_loads is not called. + */ + template + NodeGroupTerms LeaderTerms(uint32_t ng_id, + BetweenLoads &&between_loads) const + { + NodeGroupTerms terms; + if (!cc_nodes_init_.load(std::memory_order_acquire)) + { + return terms; + } + + // The candidate leader term first: when replay finishes, the cc node + // stores the leader term and then clears the candidate leader term. A + // load that sees the candidate leader term cleared is therefore + // followed by a load that sees the leader term set. In the opposite + // order, both loads can miss the term the node holds. + terms.candidate_ = + candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire); + between_loads(); + terms.leader_ = + leader_term_cache_[ng_id].load(std::memory_order_acquire); + return terms; + } /** * @brief Updates the leader cache of all cc node groups. @@ -913,7 +960,29 @@ class Sharder */ int64_t ng_leader_term_cache_[1000]; + /** + * @brief Per node group, the term in which this node serves the group: set + * when replay of every log group has finished + * (CcNode::FinishLogGroupReplay()), or at once when the node runs without + * a log service; -1 when this node does not serve the group. + * + * This node's own state, unlike ng_leader_term_cache_, which records the + * leader of every node group as this node knows it. Requests of ordinary + * transactions are fenced by this term, so they cannot see a group whose + * replay is still running. + */ std::atomic leader_term_cache_[1000]; + /** + * @brief Per node group, the term in which this node has been elected + * leader of the group and is replaying its log: set at leader start, + * cleared (-1) once the leader term is set, or on stepdown. + * + * It holds the same term that leader_term_cache_ takes when replay + * finishes; the two mark the stages of one leadership. Log replay and the + * requests of recovered transactions are fenced by this term, since they + * run before the group may serve. Code that needs both terms reads them + * through LeaderTerms(), which loads this one first. + */ std::atomic candidate_leader_term_cache_[1000]; // The term that standby is subscribed to. Only used on standby node. diff --git a/tx_service/include/tx_execution.h b/tx_service/include/tx_execution.h index 70ced1c9e..d481f7a26 100644 --- a/tx_service/include/tx_execution.h +++ b/tx_service/include/tx_execution.h @@ -272,21 +272,26 @@ class TransactionExecution : public LinkedTransaction /** * @brief Check if current node is still leader of the transaction owner * node group. + * + * Either the leader term matches the transaction's term, or the + * transaction is recovering and the candidate leader term matches it. The + * candidate leader term must match too: if this node loses the group and + * is elected again before a recovered transaction finishes, the replay in + * the new term recovers the transaction again, and the one from the old + * term must stop. + * + * @return true if the transaction may go on running on this node. */ bool CheckLeaderTerm() const { - NodeGroupId ng_id = TxCcNodeId(); - if (TxStatus() != TxnStatus::Recovering) - { - return Sharder::Instance().CheckLeaderTerm(ng_id, TxTerm()); - } - - // A recovered transaction is also valid while the node is still the - // candidate leader. Both terms come from one read, so that the check - // does not fail at the moment replay finishes. - const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id); + // Both terms come from one LeaderTerms() read. Read one after the + // other, a recovered transaction checked at the moment replay + // finishes could see neither term. + const NodeGroupTerms terms = + Sharder::Instance().LeaderTerms(TxCcNodeId()); return (terms.leader_ >= 0 && terms.leader_ == TxTerm()) || - terms.candidate_ >= 0; + (TxStatus() == TxnStatus::Recovering && terms.candidate_ >= 0 && + terms.candidate_ == TxTerm()); } bool CheckStandbyTerm() const diff --git a/tx_service/src/fault/cc_node.cpp b/tx_service/src/fault/cc_node.cpp index c91a08729..c0de14bda 100644 --- a/tx_service/src/fault/cc_node.cpp +++ b/tx_service/src/fault/cc_node.cpp @@ -137,12 +137,13 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id, int64_t CcNode::PinData() { std::unique_lock lk(pinning_threads_mux_); - // The leader term, or the candidate leader term while the node is - // replaying. The two are read as one pair: read one after the other, both - // can be missed at the moment replay finishes, and the caller would be - // told that this node holds no term. + // Returns the leader term if this node leads the group. While this node + // is still replaying the group's log, it has no leader term yet, so the + // candidate leader term is returned instead. Both terms come from one + // LeaderTerms() read: read one after the other, a call at the moment + // replay finishes could see neither term and wrongly return -1. const int64_t leader_term = - Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate(); + Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm(); if (leader_term > 0) { pinning_threads_++; @@ -978,7 +979,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, while (cntl.Failed() || start_follow_resp.error()) { - if (Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate() > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 || Sharder::Instance().StandbyNodeTerm() > 0 || Sharder::Instance().CandidateStandbyNodeTerm() > 0) { @@ -1042,7 +1043,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, int64_t standby_term = (primary_term << 32) | start_follow_resp.subscribe_id(); // verify we're not on a newer term. - if (Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate() > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 || Sharder::Instance().StandbyNodeTerm() > standby_term || Sharder::Instance().CandidateStandbyNodeTerm() > standby_term) { @@ -1259,7 +1260,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, } // We only need to retry if the message is not delivered. - if (Sharder::Instance().LeaderTerms(ng_id_).LeaderOrCandidate() > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 || (Sharder::Instance().StandbyNodeTerm() != standby_term && Sharder::Instance().CandidateStandbyNodeTerm() != standby_term)) { diff --git a/tx_service/src/sharder.cpp b/tx_service/src/sharder.cpp index ee13a6f6b..5d4eacbdb 100644 --- a/tx_service/src/sharder.cpp +++ b/tx_service/src/sharder.cpp @@ -636,25 +636,6 @@ int64_t Sharder::CandidateLeaderTerm(uint32_t ng_id) const return candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire); } -NodeGroupTerms Sharder::LeaderTerms(uint32_t ng_id) const -{ - NodeGroupTerms terms; - if (!cc_nodes_init_.load(std::memory_order_acquire)) - { - return terms; - } - - // The candidate leader term first: when replay finishes, the cc node - // stores the leader term and then clears the candidate leader term. A - // load that sees the candidate leader term cleared is therefore followed - // by a load that sees the leader term set. In the opposite order, both - // loads can miss the term the node holds. - terms.candidate_ = - candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire); - terms.leader_ = leader_term_cache_[ng_id].load(std::memory_order_acquire); - return terms; -} - void Sharder::UpdateLeaders() { std::shared_lock cnf_lk(cluster_cnf_mux_); @@ -701,7 +682,7 @@ int64_t Sharder::FetchLeaderTerm(uint32_t ng_id, uint32_t node_id) { if (node_id == node_id_) { - return LeaderTerms(ng_id).LeaderOrCandidate(); + return LeaderTerms(ng_id).LeadershipTerm(); } std::shared_ptr channel = GetCcNodeServiceChannel(node_id); if (channel == nullptr) diff --git a/tx_service/tests/CMakeLists.txt b/tx_service/tests/CMakeLists.txt index 10ccc14a1..1574c1272 100644 --- a/tx_service/tests/CMakeLists.txt +++ b/tx_service/tests/CMakeLists.txt @@ -104,6 +104,7 @@ set(OWN_MAIN_TESTS MemDataStore-Test TxConsistency-Test TestNodeSmoke-Test + LeaderTerms-Test ) function(add_substrate_test name catch_lib) diff --git a/tx_service/tests/LeaderTerms-Test.cpp b/tx_service/tests/LeaderTerms-Test.cpp new file mode 100644 index 000000000..ebf297473 --- /dev/null +++ b/tx_service/tests/LeaderTerms-Test.cpp @@ -0,0 +1,235 @@ +/** + * @file LeaderTerms-Test.cpp + * @brief Deterministic checks of the leader term and the candidate leader + * term a node reads for a node group while it passes from candidate leader to + * leader, and of which transactions TransactionExecution::CheckLeaderTerm() + * accepts for each pair of terms. + * + * The checks set the terms of the test node's own node group directly, the way + * the cc node sets them at leader start, replay finish and stepdown, and + * restore them before the node shuts down. + */ +#include "harness/test_node.h" +#include "sharder.h" +#include "tx_execution.h" +#include "tx_util.h" + +// After the engine's headers: Catch2's CHECK must replace the glog CHECK they +// bring in. +#include +#include +#include + +using namespace txservice; +using namespace txservice::test; + +namespace +{ +/** + * @brief Sets this node's leader term and candidate leader term of a node + * group. + * + * @param ng_id The node group. + * @param leader The leader term; -1 if this node does not lead the group. + * @param candidate The candidate leader term; -1 if this node is not replaying + * the group's log. + */ +void SetTerms(NodeGroupId ng_id, int64_t leader, int64_t candidate) +{ + Sharder::Instance().SetLeaderTerm(ng_id, leader); + Sharder::Instance().SetCandidateTerm(ng_id, candidate); +} + +/** + * @brief Promotes this node from candidate leader to leader of a node group, + * in the order CcNode::FinishLogGroupReplay() uses: the leader term is stored + * before the candidate leader term is cleared. + * + * @param ng_id The node group. + * @param term The term in which this node was elected. + */ +void Promote(NodeGroupId ng_id, int64_t term) +{ + Sharder::Instance().SetLeaderTerm(ng_id, term); + Sharder::Instance().SetCandidateTerm(ng_id, -1); +} + +/** + * @brief Restores a node group's terms when the check ends, also when an + * assertion fails, so the test node shuts down as the leader it started as. + */ +struct TermsRestorer +{ + ~TermsRestorer() + { + SetTerms(ng_id_, terms_.leader_, terms_.candidate_); + } + + NodeGroupId ng_id_; ///< The node group whose terms are restored. + NodeGroupTerms terms_; ///< The terms to restore. +}; + +/** + * @brief A transaction number whose coordinator is the given node group, as + * TransactionExecution::TxCcNodeId() decodes it. + * + * @param ng_id The node group. + * @return The transaction number. + */ +uint64_t TxnOfNodeGroup(NodeGroupId ng_id) +{ + return (static_cast(ng_id) << 42) | 1; +} + +/** + * @brief The single test node of this process; main() shuts it down after the + * test cases have run, before static destruction begins. + * + * The engine's Sharder is a process-wide singleton whose RPC services register + * once, so a process can bring up only one TestNode (see + * TestNodeSmoke-Test.cpp). Both test cases share this one. + */ +std::unique_ptr test_node; + +/** + * @brief The single test node of this process, brought up on first use, so + * that listing the test cases does not start a node. + * + * @return The test node. + */ +TestNode &Node() +{ + if (test_node == nullptr) + { + test_node = std::make_unique(TestNodeOptions{}.CoreNum(1)); + } + return *test_node; +} +} // namespace + +TEST_CASE( + "LeaderTerms never misses the term when the candidate leader is " + "promoted between its two loads", + "[leader_terms]") +{ + Node(); + Sharder &sharder = Sharder::Instance(); + const NodeGroupId ng_id = sharder.NativeNodeGroup(); + const NodeGroupTerms initial = sharder.LeaderTerms(ng_id); + REQUIRE(initial.leader_ > 0); + REQUIRE(initial.candidate_ < 0); + TermsRestorer restorer{ng_id, initial}; + const int64_t term = initial.leader_ + 1; + + // The promotion lands between the two loads. Read leader term first, + // the pair would be (-1, -1): the leader term loaded before it was set, + // the candidate leader term loaded after it was cleared. + SetTerms(ng_id, -1, term); + NodeGroupTerms terms = + sharder.LeaderTerms(ng_id, [&] { Promote(ng_id, term); }); + CHECK(terms.candidate_ == term); + CHECK(terms.leader_ == term); + CHECK(terms.LeadershipTerm() == term); + CHECK_FALSE(terms.Replaying()); + + // The promotion lands after both loads: the node is still replaying. + SetTerms(ng_id, -1, term); + terms = sharder.LeaderTerms(ng_id); + CHECK(terms.candidate_ == term); + CHECK(terms.leader_ < 0); + CHECK(terms.LeadershipTerm() == term); + CHECK(terms.Replaying()); + + // The promotion lands before both loads: the node leads the group. + Promote(ng_id, term); + terms = sharder.LeaderTerms(ng_id); + CHECK(terms.candidate_ < 0); + CHECK(terms.leader_ == term); + CHECK(terms.LeadershipTerm() == term); + CHECK_FALSE(terms.Replaying()); +} + +TEST_CASE( + "CheckLeaderTerm accepts a recovered transaction only in its own " + "term", + "[leader_terms]") +{ + Node(); + Sharder &sharder = Sharder::Instance(); + const NodeGroupId ng_id = sharder.NativeNodeGroup(); + const NodeGroupTerms initial = sharder.LeaderTerms(ng_id); + REQUIRE(initial.leader_ > 0); + REQUIRE(initial.candidate_ < 0); + const int64_t term = initial.leader_; + + // A transaction that replay recovered in the node's current term. + TransactionExecution *recovered = Node().Service()->NewTx(); + recovered->SetRecoverTxState(TxnOfNodeGroup(ng_id), term, 1); + REQUIRE(recovered->TxStatus() == TxnStatus::Recovering); + REQUIRE(recovered->TxCcNodeId() == ng_id); + + // An ordinary transaction, begun while the node leads the group. + TxHandle ordinary = Node().BeginTx(); + REQUIRE(ordinary.Txm()->TxTerm() == term); + + { + TermsRestorer restorer{ng_id, initial}; + + SECTION("the node leads the group in the transaction's term") + { + SetTerms(ng_id, term, -1); + CHECK(recovered->CheckLeaderTerm()); + CHECK(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node replays the group's log in the transaction's term") + { + // Only a recovered transaction runs before replay has finished. + SetTerms(ng_id, -1, term); + CHECK(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node lost the group and replays it in a newer term") + { + // The newer term's replay recovers the transaction again; the + // one recovered in the older term must stop. + SetTerms(ng_id, -1, term + 1); + CHECK_FALSE(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node lost the group and leads it in a newer term") + { + SetTerms(ng_id, term + 1, -1); + CHECK_FALSE(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node neither leads the group nor replays its log") + { + SetTerms(ng_id, -1, -1); + CHECK_FALSE(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + } + + // Both transactions end in the term they were begun in. + txservice::AbortTx(recovered); + CHECK(ordinary.Commit()); +} + +/** + * @brief Runs the Catch2 session, then shuts the test node down if a test + * case brought it up. + * + * @param argc Argument count. + * @param argv Arguments. + * @return Catch2's exit code. + */ +int main(int argc, char **argv) +{ + const int result = Catch::Session().run(argc, argv); + test_node.reset(); + return result; +}