Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions docs/06-distribution-and-clustering.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,13 @@ NG's only replay request to the log groups
every node the NG, its new leader and the term (`NotifyNewLeaderStart`, best
effort). The leader term is
published only after every log group has completed replay, so normal CC
requests cannot observe a partially recovered in-memory replica. A synchronized
standby may retain its warm state and replay only the missing tail; a cold
candidate rebuilds from storage and WAL.
requests cannot observe a partially recovered in-memory replica. The candidate
term is cleared only after the leader term is published, so from election to
stepdown at least one of the two is set. They are separate values; code that
needs both reads them as one pair (`Sharder::LeaderTerms()`), candidate term
first, and so never finds both unset while the node passes from candidate to
leader. A synchronized standby may retain its warm state and replay only the
missing tail; a cold candidate rebuilds from storage and WAL.

On stepdown, the serving and candidate terms are invalidated before NG-owned
memory is released. Code that accesses such memory outside the shard lifecycle
Expand Down Expand Up @@ -168,6 +172,7 @@ candidate becomes a complete in-memory replica.
| Cluster topology, placement functions, leader caches, terms, and data pinning are owned by `Sharder` | `tx_service/include/sharder.h`; `tx_service/src/sharder.cpp` |
| Election and membership cross the host-manager RPC boundary | `tx_service/include/proto/cc_request.proto`; `tx_service/src/remote/cc_node_service.cpp`; `tx_service/src/sharder.cpp` |
| An NG becomes serviceable only after leader recovery completes | `tx_service/include/fault/cc_node.h`; `tx_service/src/fault/cc_node.cpp`; `tx_service/src/fault/log_replay_service.cpp` |
| The leader term and the candidate term are read as one pair, candidate term first | `tx_service/include/sharder.h` (`NodeGroupTerms`, `Sharder::LeaderTerms`); `tx_service/src/fault/cc_node.cpp` (`FinishLogGroupReplay`) |
| Local and remote requests converge on the CC-request/shard execution model | `tx_service/include/remote/remote_cc_handler.h`; `tx_service/include/remote/remote_cc_request.h`; `tx_service/src/remote/cc_stream_sender.cpp`; `tx_service/src/remote/cc_stream_receiver.cpp` |
| Cluster configuration and bucket ownership are transactional metadata | `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/include/cluster_config_record.h`; `tx_service/include/cc/range_bucket_cc_map.h`; `tx_service/src/cc/local_cc_shards.cpp` |
| Scaling and data migration are resumable transaction operations | `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp`; `tx_service/tx-log-protos/log.proto` |
Expand Down
19 changes: 13 additions & 6 deletions tx_service/include/cc/catalog_cc_map.h
Original file line number Diff line number Diff line change
Expand Up @@ -322,14 +322,21 @@ class CatalogCcMap
assert(req.IsLocal());

uint32_t ng_id = req.NodeGroupId();
int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id);
ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm());

int64_t ng_term = -1;
if (req.IsInRecovering())
{
ng_term = ng_term > 0
? ng_term
: Sharder::Instance().CandidateLeaderTerm(ng_id);
// A recovering request is also served in the candidate leader
// term. The leader term and the candidate leader term come from
// one read, so that neither is missed when replay finishes.
const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id);
ng_term =
std::max(terms.leader_, Sharder::Instance().StandbyNodeTerm());
ng_term = ng_term > 0 ? ng_term : terms.candidate_;
}
else
{
ng_term = std::max(Sharder::Instance().LeaderTerm(ng_id),
Sharder::Instance().StandbyNodeTerm());
}

CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_ReadCc", {
Expand Down
5 changes: 3 additions & 2 deletions tx_service/include/cc/range_bucket_cc_map.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,15 @@ class RangeBucketCcMap

assert(req.IsLocal());
uint32_t ng_id = req.NodeGroupId();
int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id);
int64_t ng_term = -1;
if (req.IsInRecovering())
{
ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id);
}
else
{
ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm());
ng_term = std::max(Sharder::Instance().LeaderTerm(ng_id),
Sharder::Instance().StandbyNodeTerm());
}

if (ng_term < 0)
Expand Down
130 changes: 130 additions & 0 deletions tx_service/include/sharder.h
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,54 @@ struct ClusterConfig
std::shared_ptr<std::unordered_map<NodeId, NodeConfig>> nodes_configs_;
};

/**
* @brief The leader term and the candidate leader term of a cc node group on
* this node, read together by Sharder::LeaderTerms().
*/
struct NodeGroupTerms
{
/**
* @brief Whether this node is replaying the node group's log and does not
* lead the group yet.
*
* @return true if the candidate leader term is set and the leader term is
* not.
*/
bool Replaying() const
{
return leader_ < 0 && candidate_ > 0;
}

/**
* @brief The leader term if this node leads the node group; otherwise the
* candidate leader term if this node is still replaying the group's log.
*
* The candidate leader term and the leader term are one term at two
* stages: a node elected in term T publishes T as its candidate leader
* term while it replays the group's log, and publishes the same T as its
* leader term when replay finishes (CcNode::FinishLogGroupReplay()).
*
* @return The leader term if it is positive, else the candidate leader
* term; negative if this node neither leads the group nor replays its
* log.
*/
int64_t LeadershipTerm() const
{
return leader_ > 0 ? leader_ : candidate_;
}

/**
* The term in which this node leads the node group; negative if it does
* not.
*/
int64_t leader_{-1};
/**
* The term in which this node is replaying the node group's log before it
* leads the group; negative if it is not.
*/
int64_t candidate_{-1};
};

/**
* Sharder is the collection of services supplied by TxService, which includes:
* 1. cc_stream rpc service, which transfers CcRequests between different
Expand Down Expand Up @@ -348,6 +396,66 @@ class Sharder

int64_t CandidateLeaderTerm(uint32_t ng_id) const;

/**
* @brief Reads the leader term and the candidate leader term of a cc node
* group on this node as one consistent pair.
*
* The two terms are separate atomics. When log replay finishes, the cc
* node publishes the leader term and only then clears the candidate
* leader term (CcNode::FinishLogGroupReplay()). This function loads the
* candidate leader term first and the leader term second, the reverse of
* that order, so the pair it returns never shows both terms unset while
* the node passes from candidate leader to leader. Code that looks at
* both terms must call this function: LeaderTerm() followed by
* CandidateLeaderTerm() can load the leader term before it is set and
* the candidate leader term after it is cleared, and conclude that the
* node is neither leader nor candidate leader.
*
* @param ng_id The cc node group ID.
* @return The two terms; both negative before the cc nodes are
* initialized.
*/
NodeGroupTerms LeaderTerms(uint32_t ng_id) const
{
return LeaderTerms(ng_id, [] {});
}

/**
* @brief LeaderTerms(ng_id), calling a function between the two loads.
*
* For tests: the function can change the terms at the one point where
* the order of the loads matters, for example promote the candidate
* leader the way CcNode::FinishLogGroupReplay() does.
*
* @param ng_id The cc node group ID.
* @param between_loads Called after the candidate leader term is loaded
* and before the leader term is loaded.
* @return The two terms; both negative before the cc nodes are
* initialized, in which case between_loads is not called.
*/
template <typename BetweenLoads>
NodeGroupTerms LeaderTerms(uint32_t ng_id,
BetweenLoads &&between_loads) const
{
NodeGroupTerms terms;
if (!cc_nodes_init_.load(std::memory_order_acquire))
{
return terms;
}

// The candidate leader term first: when replay finishes, the cc node
// stores the leader term and then clears the candidate leader term. A
// load that sees the candidate leader term cleared is therefore
// followed by a load that sees the leader term set. In the opposite
// order, both loads can miss the term the node holds.
terms.candidate_ =
candidate_leader_term_cache_[ng_id].load(std::memory_order_acquire);
between_loads();
terms.leader_ =
leader_term_cache_[ng_id].load(std::memory_order_acquire);
return terms;
}

/**
* @brief Updates the leader cache of all cc node groups.
*
Expand Down Expand Up @@ -852,7 +960,29 @@ class Sharder
*/
int64_t ng_leader_term_cache_[1000];

/**
* @brief Per node group, the term in which this node serves the group: set
* when replay of every log group has finished
* (CcNode::FinishLogGroupReplay()), or at once when the node runs without
* a log service; -1 when this node does not serve the group.
*
* This node's own state, unlike ng_leader_term_cache_, which records the
* leader of every node group as this node knows it. Requests of ordinary
* transactions are fenced by this term, so they cannot see a group whose
* replay is still running.
*/
std::atomic<int64_t> leader_term_cache_[1000];
/**
* @brief Per node group, the term in which this node has been elected
* leader of the group and is replaying its log: set at leader start,
* cleared (-1) once the leader term is set, or on stepdown.
*
* It holds the same term that leader_term_cache_ takes when replay
* finishes; the two mark the stages of one leadership. Log replay and the
* requests of recovered transactions are fenced by this term, since they
* run before the group may serve. Code that needs both terms reads them
* through LeaderTerms(), which loads this one first.
*/
std::atomic<int64_t> candidate_leader_term_cache_[1000];

// The term that standby is subscribed to. Only used on standby node.
Expand Down
26 changes: 17 additions & 9 deletions tx_service/include/tx_execution.h
Original file line number Diff line number Diff line change
Expand Up @@ -272,18 +272,26 @@ class TransactionExecution : public LinkedTransaction
/**
* @brief Check if current node is still leader of the transaction owner
* node group.
*
* Either the leader term matches the transaction's term, or the
* transaction is recovering and the candidate leader term matches it. The
* candidate leader term must match too: if this node loses the group and
* is elected again before a recovered transaction finishes, the replay in
* the new term recovers the transaction again, and the one from the old
* term must stop.
*
* @return true if the transaction may go on running on this node.
*/
bool CheckLeaderTerm() const
{
NodeGroupId ng_id = TxCcNodeId();
if (Sharder::Instance().CheckLeaderTerm(ng_id, TxTerm()) ||
(TxStatus() == TxnStatus::Recovering &&
Sharder::Instance().CandidateLeaderTerm(ng_id) >= 0))
{
return true;
}

return false;
// Both terms come from one LeaderTerms() read. Read one after the
// other, a recovered transaction checked at the moment replay
// finishes could see neither term.
const NodeGroupTerms terms =
Sharder::Instance().LeaderTerms(TxCcNodeId());
return (terms.leader_ >= 0 && terms.leader_ == TxTerm()) ||
(TxStatus() == TxnStatus::Recovering && terms.candidate_ >= 0 &&
terms.candidate_ == TxTerm());
}

bool CheckStandbyTerm() const
Expand Down
8 changes: 3 additions & 5 deletions tx_service/src/cc/cc_req_misc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,9 @@ bool FetchCatalogCc::ValidTermCheck()
}
else
{
int64_t cc_ng_term = Sharder::Instance().LeaderTerm(cc_ng_id_);
if (cc_ng_term < 0)
{
cc_ng_term = Sharder::Instance().CandidateLeaderTerm(cc_ng_id_);
}
const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(cc_ng_id_);
const int64_t cc_ng_term =
terms.leader_ < 0 ? terms.candidate_ : terms.leader_;

if (cc_ng_term != cc_ng_term_)
{
Expand Down
5 changes: 3 additions & 2 deletions tx_service/src/cc/local_cc_shards.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1751,8 +1751,9 @@ uint64_t LocalCcShards::CountSlices(const TableName &table_name,
TableName range_table_name(table_name.StringView(),
TableType::RangePartition,
table_name.Engine());
assert(Sharder::Instance().LeaderTerm(local_ng_id) >= 0 ||
Sharder::Instance().CandidateLeaderTerm(local_ng_id) >= 0);
[[maybe_unused]] const NodeGroupTerms local_ng_terms =
Sharder::Instance().LeaderTerms(local_ng_id);
assert(local_ng_terms.leader_ >= 0 || local_ng_terms.candidate_ >= 0);

uint64_t slices = 0;

Expand Down
4 changes: 2 additions & 2 deletions tx_service/src/checkpointer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,8 @@ bool Checkpointer::IsCurrentCheckpointTerm(NodeGroupId node_group_id,
}

Sharder &sharder = Sharder::Instance();
if (sharder.LeaderTerm(node_group_id) == term ||
sharder.CandidateLeaderTerm(node_group_id) == term)
const NodeGroupTerms terms = sharder.LeaderTerms(node_group_id);
if (terms.leader_ == term || terms.candidate_ == term)
{
return true;
}
Expand Down
35 changes: 16 additions & 19 deletions tx_service/src/fault/cc_node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,12 @@ CcNode::CcNode(const uint32_t ng_id,
bool CcNode::CheckLogGroupReplayFinished(uint32_t log_group_id, int64_t ng_term)
{
std::lock_guard<std::mutex> lk(recovery_mux_);
int64_t term = Sharder::Instance().LeaderTerm(ng_id_);
if (term >= ng_term)
const NodeGroupTerms terms = Sharder::Instance().LeaderTerms(ng_id_);
if (terms.leader_ >= ng_term)
{
return true;
}
int64_t candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id_);
if (candidate_term > ng_term)
if (terms.candidate_ > ng_term)
{
return true;
}
Expand Down Expand Up @@ -122,6 +121,9 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id,
recovered_log_groups_.clear();

candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id_);
// The leader term is set before the candidate leader term is cleared.
// Sharder::LeaderTerms() reads the two in the reverse order and relies
// on this one to never see both unset during the hand-over.
Sharder::Instance().SetLeaderTerm(ng_id_, candidate_term);
LOG(INFO) << "The leader of cc node group ng#" << ng_id_
<< " with the term " << candidate_term
Expand All @@ -135,19 +137,17 @@ void CcNode::FinishLogGroupReplay(uint32_t log_group_id,
int64_t CcNode::PinData()
{
std::unique_lock lk(pinning_threads_mux_);
int64_t leader_term = Sharder::Instance().LeaderTerm(ng_id_);
// Returns the leader term if this node leads the group. While this node
// is still replaying the group's log, it has no leader term yet, so the
// candidate leader term is returned instead. Both terms come from one
// LeaderTerms() read: read one after the other, a call at the moment
// replay finishes could see neither term and wrongly return -1.
const int64_t leader_term =
Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm();
if (leader_term > 0)
{
pinning_threads_++;
}
else
{
leader_term = Sharder::Instance().CandidateLeaderTerm(ng_id_);
if (leader_term > 0)
{
pinning_threads_++;
}
}
return leader_term;
}

Expand Down Expand Up @@ -979,8 +979,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id,

while (cntl.Failed() || start_follow_resp.error())
{
if (Sharder::Instance().LeaderTerm(ng_id_) > 0 ||
Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 ||
if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 ||
Sharder::Instance().StandbyNodeTerm() > 0 ||
Sharder::Instance().CandidateStandbyNodeTerm() > 0)
{
Expand Down Expand Up @@ -1044,8 +1043,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id,
int64_t standby_term =
(primary_term << 32) | start_follow_resp.subscribe_id();
// verify we're not on a newer term.
if (Sharder::Instance().LeaderTerm(ng_id_) > 0 ||
Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 ||
if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 ||
Sharder::Instance().StandbyNodeTerm() > standby_term ||
Sharder::Instance().CandidateStandbyNodeTerm() > standby_term)
{
Expand Down Expand Up @@ -1262,8 +1260,7 @@ void CcNode::SubscribePrimaryNode(uint32_t leader_node_id,
}

// We only need to retry if the message is not delivered.
if (Sharder::Instance().LeaderTerm(ng_id_) > 0 ||
Sharder::Instance().CandidateLeaderTerm(ng_id_) > 0 ||
if (Sharder::Instance().LeaderTerms(ng_id_).LeadershipTerm() > 0 ||
(Sharder::Instance().StandbyNodeTerm() != standby_term &&
Sharder::Instance().CandidateStandbyNodeTerm() != standby_term))
{
Expand Down
Loading
Loading