diff --git a/docs/06-distribution-and-clustering.md b/docs/06-distribution-and-clustering.md index 38fc7edf..6efc0c25 100644 --- a/docs/06-distribution-and-clustering.md +++ b/docs/06-distribution-and-clustering.md @@ -83,9 +83,13 @@ NG's only replay request to the log groups every node the NG, its new leader and the term (`NotifyNewLeaderStart`, best effort). The leader term is published only after every log group has completed replay, so normal CC -requests cannot observe a partially recovered in-memory replica. A synchronized -standby may retain its warm state and replay only the missing tail; a cold -candidate rebuilds from storage and WAL. +requests cannot observe a partially recovered in-memory replica. The candidate +term is cleared only after the leader term is published, so from election to +stepdown at least one of the two is set. They are separate values; code that +needs both reads them as one pair (`Sharder::LeaderTerms()`), candidate term +first, and so never finds both unset while the node passes from candidate to +leader. A synchronized standby may retain its warm state and replay only the +missing tail; a cold candidate rebuilds from storage and WAL. On stepdown, the serving and candidate terms are invalidated before NG-owned memory is released. Code that accesses such memory outside the shard lifecycle @@ -168,6 +172,7 @@ candidate becomes a complete in-memory replica. | Cluster topology, placement functions, leader caches, terms, and data pinning are owned by `Sharder` | `tx_service/include/sharder.h`; `tx_service/src/sharder.cpp` | | Election and membership cross the host-manager RPC boundary | `tx_service/include/proto/cc_request.proto`; `tx_service/src/remote/cc_node_service.cpp`; `tx_service/src/sharder.cpp` | | An NG becomes serviceable only after leader recovery completes | `tx_service/include/fault/cc_node.h`; `tx_service/src/fault/cc_node.cpp`; `tx_service/src/fault/log_replay_service.cpp` | +| The leader term and the candidate term are read as one pair, candidate term first | `tx_service/include/sharder.h` (`NodeGroupTerms`, `Sharder::LeaderTerms`); `tx_service/src/fault/cc_node.cpp` (`FinishLogGroupReplay`) | | Local and remote requests converge on the CC-request/shard execution model | `tx_service/include/remote/remote_cc_handler.h`; `tx_service/include/remote/remote_cc_request.h`; `tx_service/src/remote/cc_stream_sender.cpp`; `tx_service/src/remote/cc_stream_receiver.cpp` | | Cluster configuration and bucket ownership are transactional metadata | `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/include/cluster_config_record.h`; `tx_service/include/cc/range_bucket_cc_map.h`; `tx_service/src/cc/local_cc_shards.cpp` | | Scaling and data migration are resumable transaction operations | `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp`; `tx_service/tx-log-protos/log.proto` | diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 2ce22163..cea7f6d0 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -322,14 +322,21 @@ class CatalogCcMap assert(req.IsLocal()); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); - + int64_t ng_term = -1; if (req.IsInRecovering()) { - ng_term = ng_term > 0 - ? ng_term - : Sharder::Instance().CandidateLeaderTerm(ng_id); + // A recovering request is also served in the candidate leader + // term. The leader term and the candidate leader term come from + // one read, so that neither is missed when replay finishes. + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id); + ng_term = + std::max(terms.leader_, Sharder::Instance().StandbyNodeTerm()); + ng_term = ng_term > 0 ? ng_term : terms.candidate_; + } + else + { + ng_term = std::max(Sharder::Instance().LeaderTerm(ng_id), + Sharder::Instance().StandbyNodeTerm()); } CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_ReadCc", { diff --git a/tx_service/include/cc/range_bucket_cc_map.h b/tx_service/include/cc/range_bucket_cc_map.h index 6805aa5f..a2ecff21 100644 --- a/tx_service/include/cc/range_bucket_cc_map.h +++ b/tx_service/include/cc/range_bucket_cc_map.h @@ -90,14 +90,15 @@ class RangeBucketCcMap assert(req.IsLocal()); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); + int64_t ng_term = -1; if (req.IsInRecovering()) { ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id); } else { - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); + ng_term = std::max(Sharder::Instance().LeaderTerm(ng_id), + Sharder::Instance().StandbyNodeTerm()); } if (ng_term < 0) diff --git a/tx_service/include/sharder.h b/tx_service/include/sharder.h index eccbb1d0..c925bc05 100644 --- a/tx_service/include/sharder.h +++ b/tx_service/include/sharder.h @@ -157,6 +157,54 @@ struct ClusterConfig std::shared_ptr> nodes_configs_; }; +/** + * @brief The leader term and the candidate leader term of a cc node group on + * this node, read together by Sharder::LeaderTerms(). + */ +struct NodeGroupTerms +{ + /** + * @brief Whether this node is replaying the node group's log and does not + * lead the group yet. + * + * @return true if the candidate leader term is set and the leader term is + * not. + */ + bool Replaying() const + { + return leader_ < 0 && candidate_ > 0; + } + + /** + * @brief The leader term if this node leads the node group; otherwise the + * candidate leader term if this node is still replaying the group's log. + * + * The candidate leader term and the leader term are one term at two + * stages: a node elected in term T publishes T as its candidate leader + * term while it replays the group's log, and publishes the same T as its + * leader term when replay finishes (CcNode::FinishLogGroupReplay()). + * + * @return The leader term if it is positive, else the candidate leader + * term; negative if this node neither leads the group nor replays its + * log. + */ + int64_t LeadershipTerm() const + { + return leader_ > 0 ? leader_ : candidate_; + } + + /** + * The term in which this node leads the node group; negative if it does + * not. + */ + int64_t leader_{-1}; + /** + * The term in which this node is replaying the node group's log before it + * leads the group; negative if it is not. + */ + int64_t candidate_{-1}; +}; + /** * Sharder is the collection of services supplied by TxService, which includes: * 1. cc_stream rpc service, which transfers CcRequests between different @@ -348,6 +396,66 @@ class Sharder int64_t CandidateLeaderTerm(uint32_t ng_id) const; + /** + * @brief Reads the leader term and the candidate leader term of a cc node + * group on this node as one consistent pair. + * + * The two terms are separate atomics. When log replay finishes, the cc + * node publishes the leader term and only then clears the candidate + * leader term (CcNode::FinishLogGroupReplay()). This function loads the + * candidate leader term first and the leader term second, the reverse of + * that order, so the pair it returns never shows both terms unset while + * the node passes from candidate leader to leader. Code that looks at + * both terms must call this function: LeaderTerm() followed by + * CandidateLeaderTerm() can load the leader term before it is set and + * the candidate leader term after it is cleared, and conclude that the + * node is neither leader nor candidate leader. + * + * @param ng_id The cc node group ID. + * @return The two terms; both negative before the cc nodes are + * initialized. + */ + NodeGroupTerms LeaderTerms(uint32_t ng_id) const + { + return LeaderTerms(ng_id, [] {}); + } + + /** + * @brief LeaderTerms(ng_id), calling a function between the two loads. + * + * For tests: the function can change the terms at the one point where + * the order of the loads matters, for example promote the candidate + * leader the way CcNode::FinishLogGroupReplay() does. + * + * @param ng_id The cc node group ID. + * @param between_loads Called after the candidate leader term is loaded + * and before the leader term is loaded. + * @return The two terms; both negative before the cc nodes are + * initialized, in which case between_loads is not called. + */ + template + NodeGroupTerms LeaderTerms(uint32_t ng_id, + BetweenLoads &&between_loads) const + { + NodeGroupTerms terms; + if (!cc_nodes_init_.load(std::memory_order_acquire)) + { + return terms; + } + + // The candidate leader term first: when replay finishes, the cc node + // stores the leader term and then clears the candidate leader term. A + // load that sees the candidate leader term cleared is therefore + // followed by a load that sees the leader term set. In the opposite + // order, both loads can miss the term the node holds. + terms.candidate_ = + candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire); + between_loads(); + terms.leader_ = + leader_term_cache_[ng_id].load(std::memory_order_acquire); + return terms; + } + /** * @brief Updates the leader cache of all cc node groups. * @@ -852,7 +960,29 @@ class Sharder */ int64_t ng_leader_term_cache_[1000]; + /** + * @brief Per node group, the term in which this node serves the group: set + * when replay of every log group has finished + * (CcNode::FinishLogGroupReplay()), or at once when the node runs without + * a log service; -1 when this node does not serve the group. + * + * This node's own state, unlike ng_leader_term_cache_, which records the + * leader of every node group as this node knows it. Requests of ordinary + * transactions are fenced by this term, so they cannot see a group whose + * replay is still running. + */ std::atomic leader_term_cache_[1000]; + /** + * @brief Per node group, the term in which this node has been elected + * leader of the group and is replaying its log: set at leader start, + * cleared (-1) once the leader term is set, or on stepdown. + * + * It holds the same term that leader_term_cache_ takes when replay + * finishes; the two mark the stages of one leadership. Log replay and the + * requests of recovered transactions are fenced by this term, since they + * run before the group may serve. Code that needs both terms reads them + * through LeaderTerms(), which loads this one first. + */ std::atomic candidate_leader_term_cache_[1000]; // The term that standby is subscribed to. Only used on standby node. diff --git a/tx_service/include/tx_execution.h b/tx_service/include/tx_execution.h index c7087317..d481f7a2 100644 --- a/tx_service/include/tx_execution.h +++ b/tx_service/include/tx_execution.h @@ -272,18 +272,26 @@ class TransactionExecution : public LinkedTransaction /** * @brief Check if current node is still leader of the transaction owner * node group. + * + * Either the leader term matches the transaction's term, or the + * transaction is recovering and the candidate leader term matches it. The + * candidate leader term must match too: if this node loses the group and + * is elected again before a recovered transaction finishes, the replay in + * the new term recovers the transaction again, and the one from the old + * term must stop. + * + * @return true if the transaction may go on running on this node. */ bool CheckLeaderTerm() const { - NodeGroupId ng_id = TxCcNodeId(); - if (Sharder::Instance().CheckLeaderTerm(ng_id, TxTerm()) || - (TxStatus() == TxnStatus::Recovering && - Sharder::Instance().CandidateLeaderTerm(ng_id) >= 0)) - { - return true; - } - - return false; + // Both terms come from one LeaderTerms() read. Read one after the + // other, a recovered transaction checked at the moment replay + // finishes could see neither term. + const NodeGroupTerms terms = + Sharder::Instance().LeaderTerms(TxCcNodeId()); + return (terms.leader_ >= 0 && terms.leader_ == TxTerm()) || + (TxStatus() == TxnStatus::Recovering && terms.candidate_ >= 0 && + terms.candidate_ == TxTerm()); } bool CheckStandbyTerm() const diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 9ad864f0..af6123f7 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -122,11 +122,9 @@ bool FetchCatalogCc::ValidTermCheck() } else { - int64_t cc_ng_term = Sharder::Instance().LeaderTerm(cc_ng_id_); - if (cc_ng_term < 0) - { - cc_ng_term = Sharder::Instance().CandidateLeaderTerm(cc_ng_id_); - } + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(cc_ng_id_); + const int64_t cc_ng_term = + terms.leader_ < 0 ? terms.candidate_ : terms.leader_; if (cc_ng_term != cc_ng_term_) { diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 6ef47347..19147d56 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -1751,8 +1751,9 @@ uint64_t LocalCcShards::CountSlices(const TableName &table_name, TableName range_table_name(table_name.StringView(), TableType::RangePartition, table_name.Engine()); - assert(Sharder::Instance().LeaderTerm(local_ng_id) >= 0 || - Sharder::Instance().CandidateLeaderTerm(local_ng_id) >= 0); + [[maybe_unused]] const NodeGroupTerms local_ng_terms = + Sharder::Instance().LeaderTerms(local_ng_id); + assert(local_ng_terms.leader_ >= 0 || local_ng_terms.candidate_ >= 0); uint64_t slices = 0; diff --git a/tx_service/src/checkpointer.cpp b/tx_service/src/checkpointer.cpp index 66e05de1..bea9f455 100644 --- a/tx_service/src/checkpointer.cpp +++ b/tx_service/src/checkpointer.cpp @@ -177,8 +177,8 @@ bool Checkpointer::IsCurrentCheckpointTerm(NodeGroupId node_group_id, } Sharder &sharder = Sharder::Instance(); - if (sharder.LeaderTerm(node_group_id) == term || - sharder.CandidateLeaderTerm(node_group_id) == term) + const NodeGroupTerms terms = sharder.LeaderTerms(node_group_id); + if (terms.leader_ == term || terms.candidate_ == term) { return true; } diff --git a/tx_service/src/fault/cc_node.cpp b/tx_service/src/fault/cc_node.cpp index 7ec610cc..c0de14bd 100644 --- a/tx_service/src/fault/cc_node.cpp +++ b/tx_service/src/fault/cc_node.cpp @@ -61,13 +61,12 @@ CcNode::CcNode(const uint32_t ng_id, bool CcNode::CheckLogGroupReplayFinished(uint32_t log_group_id, int64_t ng_term) { std::lock_guard lk(recovery_mux_); - int64_t term = Sharder::Instance().LeaderTerm(ng_id_); - if (term >= ng_term) + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id_); + if (terms.leader_ >= ng_term) { return true; } - int64_t candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id_); - if (candidate_term > ng_term) + if (terms.candidate_ > ng_term) { return true; } @@ -122,6 +121,9 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id, recovered_log_groups_.clear(); candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id_); + // The leader term is set before the candidate leader term is cleared. + // Sharder::LeaderTerms() reads the two in the reverse order and relies + // on this one to never see both unset during the hand-over. Sharder::Instance().SetLeaderTerm(ng_id_, candidate_term); LOG(INFO) << "The leader of cc node group ng#" << ng_id_ << " with the term " << candidate_term @@ -135,19 +137,17 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id, int64_t CcNode::PinData() { std::unique_lock lk(pinning_threads_mux_); - int64_t leader_term = Sharder::Instance().LeaderTerm(ng_id_); + // Returns the leader term if this node leads the group. While this node + // is still replaying the group's log, it has no leader term yet, so the + // candidate leader term is returned instead. Both terms come from one + // LeaderTerms() read: read one after the other, a call at the moment + // replay finishes could see neither term and wrongly return -1. + const int64_t leader_term = + Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm(); if (leader_term > 0) { pinning_threads_++; } - else - { - leader_term = Sharder::Instance().CandidateLeaderTerm(ng_id_); - if (leader_term > 0) - { - pinning_threads_++; - } - } return leader_term; } @@ -979,8 +979,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, while (cntl.Failed() || start_follow_resp.error()) { - if (Sharder::Instance().LeaderTerm(ng_id_) > 0 || - Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 || Sharder::Instance().StandbyNodeTerm() > 0 || Sharder::Instance().CandidateStandbyNodeTerm() > 0) { @@ -1044,8 +1043,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, int64_t standby_term = (primary_term << 32) | start_follow_resp.subscribe_id(); // verify we're not on a newer term. - if (Sharder::Instance().LeaderTerm(ng_id_) > 0 || - Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 || Sharder::Instance().StandbyNodeTerm() > standby_term || Sharder::Instance().CandidateStandbyNodeTerm() > standby_term) { @@ -1262,8 +1260,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id, } // We only need to retry if the message is not delivered. - if (Sharder::Instance().LeaderTerm(ng_id_) > 0 || - Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 || + if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 || (Sharder::Instance().StandbyNodeTerm() != standby_term && Sharder::Instance().CandidateStandbyNodeTerm() != standby_term)) { diff --git a/tx_service/src/remote/cc_node_service.cpp b/tx_service/src/remote/cc_node_service.cpp index 44b5b8fe..ccafb5ef 100644 --- a/tx_service/src/remote/cc_node_service.cpp +++ b/tx_service/src/remote/cc_node_service.cpp @@ -776,8 +776,7 @@ void CcNodeService::FlushDataAll(::google::protobuf::RpcController *controller, &local_shards = this->local_shards_]() { int64_t leader_term = INIT_TERM; - while (Sharder::Instance().LeaderTerm(ng_id) < 0 && - Sharder::Instance().CandidateLeaderTerm(ng_id) > 0) + while (Sharder::Instance().LeaderTerms(ng_id).Replaying()) { // The RPC server can receive the remote request, but this // node has not finish log replay, including data(.pk) log and @@ -1175,8 +1174,7 @@ void CcNodeService::GenerateSkFromPk( tx_number, tx_term]() { - while (Sharder::Instance().LeaderTerm(ng_id) < 0 && - Sharder::Instance().CandidateLeaderTerm(ng_id) > 0) + while (Sharder::Instance().LeaderTerms(ng_id).Replaying()) { // The RPC server can receive the remote request, but this // node has not finish log replay, including data(.pk) log and @@ -2186,9 +2184,9 @@ void CcNodeService::FetchNodeInfo( response->set_ng_id(ng_id); response->set_node_id(node_id); - const int64_t leader_term = Sharder::Instance().LeaderTerm(ng_id); - const int64_t candidate_term = - Sharder::Instance().CandidateLeaderTerm(ng_id); + const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id); + const int64_t leader_term = terms.leader_; + const int64_t candidate_term = terms.candidate_; if (leader_term > 0) { response->set_role(NodeRole::LeaderNode); diff --git a/tx_service/src/sharder.cpp b/tx_service/src/sharder.cpp index aa29bc9b..5d4eacbd 100644 --- a/tx_service/src/sharder.cpp +++ b/tx_service/src/sharder.cpp @@ -682,8 +682,7 @@ int64_t Sharder::FetchLeaderTerm(uint32_t ng_id, uint32_t node_id) { if (node_id == node_id_) { - const int64_t leader_term = LeaderTerm(ng_id); - return leader_term > 0 ? leader_term : CandidateLeaderTerm(ng_id); + return LeaderTerms(ng_id).LeadershipTerm(); } std::shared_ptr channel = GetCcNodeServiceChannel(node_id); if (channel == nullptr) diff --git a/tx_service/src/tx_index_operation.cpp b/tx_service/src/tx_index_operation.cpp index 0af3a01f..f0a1f130 100644 --- a/tx_service/src/tx_index_operation.cpp +++ b/tx_service/src/tx_index_operation.cpp @@ -2473,8 +2473,7 @@ void UpsertTableIndexOp::HandleRangeTask( &task_value, &dispatch_func]() { - while (Sharder::Instance().LeaderTerm(range_owner) < 0 && - Sharder::Instance().CandidateLeaderTerm(range_owner) > 0) + while (Sharder::Instance().LeaderTerms(range_owner).Replaying()) { // Waiting until log replay finished on this node // group. Including data(.pk) log and and diff --git a/tx_service/src/tx_operation.cpp b/tx_service/src/tx_operation.cpp index 3c97e10d..850ed24f 100644 --- a/tx_service/src/tx_operation.cpp +++ b/tx_service/src/tx_operation.cpp @@ -3349,16 +3349,26 @@ void SplitFlushRangeOp::ClearInfos() void SplitFlushRangeOp::Forward(TransactionExecution *txm) { - if (txm->TxStatus() == TxnStatus::Recovering && - Sharder::Instance().LeaderTerm(txm->TxCcNodeId()) < 0) + // The leader term and the candidate leader term, read as one pair. Read + // one after the other, both can be missed at the moment replay finishes, + // and the recovered split would be dropped although its term is still + // valid, leaving its prepare log and its range locks without an owner. + const NodeGroupTerms terms = + Sharder::Instance().LeaderTerms(txm->TxCcNodeId()); + if (txm->TxStatus() == TxnStatus::Recovering && terms.leader_ < 0) { // This is a recovered tx and replay is not done yet. We should wait for // replay finish before forwarding tx machine. - if (Sharder::Instance().CandidateLeaderTerm(txm->TxCcNodeId()) != - txm->TxTerm()) + if (terms.candidate_ != txm->TxTerm()) { // Recovered term is invalid. Do not call ForceToFinish as it will // cause infinite recursive call. Clean up tx state directly. + LOG(WARNING) << "Split Flush transaction recovered in term " + << txm->TxTerm() + << " is dropped, txn: " << txm->TxNumber() + << ", node group #" << txm->TxCcNodeId() + << " leader term " << terms.leader_ + << ", candidate leader term " << terms.candidate_; txm->bool_resp_->Finish(false); ClearInfos(); @@ -6415,17 +6425,27 @@ void DataMigrationOp::PrepareNextRoundBuckets() void DataMigrationOp::Forward(TransactionExecution *txm) { - if (txm->TxStatus() == TxnStatus::Recovering && - Sharder::Instance().LeaderTerm(txm->TxCcNodeId()) < 0) + // The leader term and the candidate leader term, read as one pair. Read + // one after the other, both can be missed at the moment replay finishes, + // and the recovered migration would be dropped although its term is still + // valid. + const NodeGroupTerms terms = + Sharder::Instance().LeaderTerms(txm->TxCcNodeId()); + if (txm->TxStatus() == TxnStatus::Recovering && terms.leader_ < 0) { // This is a recovered tx and replay is not done yet. We should wait // for replay finish before forwarding tx machine. - if (Sharder::Instance().CandidateLeaderTerm(txm->TxCcNodeId()) != - txm->TxTerm()) + if (terms.candidate_ != txm->TxTerm()) { // Recovered term is invalid. Do not call ForceToFinish as it // will cause infinite recursive call. Clean up tx state // directly. + LOG(WARNING) << "Data migration transaction recovered in term " + << txm->TxTerm() + << " is dropped, txn: " << txm->TxNumber() + << ", node group #" << txm->TxCcNodeId() + << " leader term " << terms.leader_ + << ", candidate leader term " << terms.candidate_; Clear(); txm->state_stack_.pop_back(); assert(txm->state_stack_.empty()); diff --git a/tx_service/tests/CMakeLists.txt b/tx_service/tests/CMakeLists.txt index ec71d203..a2abd1a4 100644 --- a/tx_service/tests/CMakeLists.txt +++ b/tx_service/tests/CMakeLists.txt @@ -106,6 +106,7 @@ set(OWN_MAIN_TESTS TestNodeSmoke-Test PayloadReservation-Test CkptCycle-Test + LeaderTerms-Test ) function(add_substrate_test name catch_lib) diff --git a/tx_service/tests/LeaderTerms-Test.cpp b/tx_service/tests/LeaderTerms-Test.cpp new file mode 100644 index 00000000..ebf29747 --- /dev/null +++ b/tx_service/tests/LeaderTerms-Test.cpp @@ -0,0 +1,235 @@ +/** + * @file LeaderTerms-Test.cpp + * @brief Deterministic checks of the leader term and the candidate leader + * term a node reads for a node group while it passes from candidate leader to + * leader, and of which transactions TransactionExecution::CheckLeaderTerm() + * accepts for each pair of terms. + * + * The checks set the terms of the test node's own node group directly, the way + * the cc node sets them at leader start, replay finish and stepdown, and + * restore them before the node shuts down. + */ +#include "harness/test_node.h" +#include "sharder.h" +#include "tx_execution.h" +#include "tx_util.h" + +// After the engine's headers: Catch2's CHECK must replace the glog CHECK they +// bring in. +#include +#include +#include + +using namespace txservice; +using namespace txservice::test; + +namespace +{ +/** + * @brief Sets this node's leader term and candidate leader term of a node + * group. + * + * @param ng_id The node group. + * @param leader The leader term; -1 if this node does not lead the group. + * @param candidate The candidate leader term; -1 if this node is not replaying + * the group's log. + */ +void SetTerms(NodeGroupId ng_id, int64_t leader, int64_t candidate) +{ + Sharder::Instance().SetLeaderTerm(ng_id, leader); + Sharder::Instance().SetCandidateTerm(ng_id, candidate); +} + +/** + * @brief Promotes this node from candidate leader to leader of a node group, + * in the order CcNode::FinishLogGroupReplay() uses: the leader term is stored + * before the candidate leader term is cleared. + * + * @param ng_id The node group. + * @param term The term in which this node was elected. + */ +void Promote(NodeGroupId ng_id, int64_t term) +{ + Sharder::Instance().SetLeaderTerm(ng_id, term); + Sharder::Instance().SetCandidateTerm(ng_id, -1); +} + +/** + * @brief Restores a node group's terms when the check ends, also when an + * assertion fails, so the test node shuts down as the leader it started as. + */ +struct TermsRestorer +{ + ~TermsRestorer() + { + SetTerms(ng_id_, terms_.leader_, terms_.candidate_); + } + + NodeGroupId ng_id_; ///< The node group whose terms are restored. + NodeGroupTerms terms_; ///< The terms to restore. +}; + +/** + * @brief A transaction number whose coordinator is the given node group, as + * TransactionExecution::TxCcNodeId() decodes it. + * + * @param ng_id The node group. + * @return The transaction number. + */ +uint64_t TxnOfNodeGroup(NodeGroupId ng_id) +{ + return (static_cast(ng_id) << 42) | 1; +} + +/** + * @brief The single test node of this process; main() shuts it down after the + * test cases have run, before static destruction begins. + * + * The engine's Sharder is a process-wide singleton whose RPC services register + * once, so a process can bring up only one TestNode (see + * TestNodeSmoke-Test.cpp). Both test cases share this one. + */ +std::unique_ptr test_node; + +/** + * @brief The single test node of this process, brought up on first use, so + * that listing the test cases does not start a node. + * + * @return The test node. + */ +TestNode &Node() +{ + if (test_node == nullptr) + { + test_node = std::make_unique(TestNodeOptions{}.CoreNum(1)); + } + return *test_node; +} +} // namespace + +TEST_CASE( + "LeaderTerms never misses the term when the candidate leader is " + "promoted between its two loads", + "[leader_terms]") +{ + Node(); + Sharder &sharder = Sharder::Instance(); + const NodeGroupId ng_id = sharder.NativeNodeGroup(); + const NodeGroupTerms initial = sharder.LeaderTerms(ng_id); + REQUIRE(initial.leader_ > 0); + REQUIRE(initial.candidate_ < 0); + TermsRestorer restorer{ng_id, initial}; + const int64_t term = initial.leader_ + 1; + + // The promotion lands between the two loads. Read leader term first, + // the pair would be (-1, -1): the leader term loaded before it was set, + // the candidate leader term loaded after it was cleared. + SetTerms(ng_id, -1, term); + NodeGroupTerms terms = + sharder.LeaderTerms(ng_id, [&] { Promote(ng_id, term); }); + CHECK(terms.candidate_ == term); + CHECK(terms.leader_ == term); + CHECK(terms.LeadershipTerm() == term); + CHECK_FALSE(terms.Replaying()); + + // The promotion lands after both loads: the node is still replaying. + SetTerms(ng_id, -1, term); + terms = sharder.LeaderTerms(ng_id); + CHECK(terms.candidate_ == term); + CHECK(terms.leader_ < 0); + CHECK(terms.LeadershipTerm() == term); + CHECK(terms.Replaying()); + + // The promotion lands before both loads: the node leads the group. + Promote(ng_id, term); + terms = sharder.LeaderTerms(ng_id); + CHECK(terms.candidate_ < 0); + CHECK(terms.leader_ == term); + CHECK(terms.LeadershipTerm() == term); + CHECK_FALSE(terms.Replaying()); +} + +TEST_CASE( + "CheckLeaderTerm accepts a recovered transaction only in its own " + "term", + "[leader_terms]") +{ + Node(); + Sharder &sharder = Sharder::Instance(); + const NodeGroupId ng_id = sharder.NativeNodeGroup(); + const NodeGroupTerms initial = sharder.LeaderTerms(ng_id); + REQUIRE(initial.leader_ > 0); + REQUIRE(initial.candidate_ < 0); + const int64_t term = initial.leader_; + + // A transaction that replay recovered in the node's current term. + TransactionExecution *recovered = Node().Service()->NewTx(); + recovered->SetRecoverTxState(TxnOfNodeGroup(ng_id), term, 1); + REQUIRE(recovered->TxStatus() == TxnStatus::Recovering); + REQUIRE(recovered->TxCcNodeId() == ng_id); + + // An ordinary transaction, begun while the node leads the group. + TxHandle ordinary = Node().BeginTx(); + REQUIRE(ordinary.Txm()->TxTerm() == term); + + { + TermsRestorer restorer{ng_id, initial}; + + SECTION("the node leads the group in the transaction's term") + { + SetTerms(ng_id, term, -1); + CHECK(recovered->CheckLeaderTerm()); + CHECK(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node replays the group's log in the transaction's term") + { + // Only a recovered transaction runs before replay has finished. + SetTerms(ng_id, -1, term); + CHECK(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node lost the group and replays it in a newer term") + { + // The newer term's replay recovers the transaction again; the + // one recovered in the older term must stop. + SetTerms(ng_id, -1, term + 1); + CHECK_FALSE(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node lost the group and leads it in a newer term") + { + SetTerms(ng_id, term + 1, -1); + CHECK_FALSE(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + + SECTION("the node neither leads the group nor replays its log") + { + SetTerms(ng_id, -1, -1); + CHECK_FALSE(recovered->CheckLeaderTerm()); + CHECK_FALSE(ordinary.Txm()->CheckLeaderTerm()); + } + } + + // Both transactions end in the term they were begun in. + txservice::AbortTx(recovered); + CHECK(ordinary.Commit()); +} + +/** + * @brief Runs the Catch2 session, then shuts the test node down if a test + * case brought it up. + * + * @param argc Argument count. + * @param argv Arguments. + * @return Catch2's exit code. + */ +int main(int argc, char **argv) +{ + const int result = Catch::Session().run(argc, argv); + test_node.reset(); + return result; +}