diff --git a/docs/03-concurrency-control.md b/docs/03-concurrency-control.md index 933dc8a07..415ec79c1 100644 --- a/docs/03-concurrency-control.md +++ b/docs/03-concurrency-control.md @@ -47,7 +47,10 @@ three stable patterns: requester to the fetch operation until backfill completes; - memory pressure transfers the request to a shard wait list while eviction or checkpoint progress makes admission possible, unless the caller requested an - immediate out-of-memory failure. + immediate out-of-memory failure. Commit-path write acquisition is an exception: + entry admission and payload-reservation failure complete with out-of-memory + instead of parking, because other keys may already hold the transaction's + reservations. In all cases, waiting occurs outside `Execute()`. A resumed request continues with the lock, fetch result, or admission opportunity granted by the component @@ -102,6 +105,16 @@ then release the write lock and wake waiters. Object-command maps use the same lock lifecycle but apply and commit engine commands against shard-owned objects. +Record writes also reserve the source record's `MemUsage()` on every destination +shard before WAL, attaching the charge to the granted write lock. Post-write +materialization, abort, and term cleanup release the charge; taking it clears it +so cleanup cannot release it twice. A write that cannot reserve completes with +out-of-memory while retaining the lock granted before the failure, so the +coordinator's abort tail releases it. Admission counts outstanding reservations, +so a shard can refuse writes while physical allocation is below the cleaning +watermark. The budget is a soft limit over periodically sampled allocator +statistics, not a hard process RSS or installation-peak guarantee. + Scanners are batched cursors over memory and, when needed, persistent data. They retain the intents or locks needed to keep continuation points valid and transfer semantic reads to the transaction read set until final validation or diff --git a/docs/04-transaction-execution.md b/docs/04-transaction-execution.md index 59ee0651b..118a76e47 100644 --- a/docs/04-transaction-execution.md +++ b/docs/04-transaction-execution.md @@ -83,8 +83,12 @@ stage that is irrelevant or was already satisfied while executing a command: 1. Read-lock the range or bucket metadata that determines placement of every write, so routing cannot change underneath commit. 2. Acquire write ownership for buffered record keys at their local or remote CC - shards. Object commands retain the owner-shard write ownership established - during execution and acquire any additional forwarding ownership they need. + shards, including the destination payload reservation. Capacity refusal + completes the acquisition with out-of-memory instead of parking; a lock + granted before the refusal stays in the acquire result so abort cleanup + releases it ([03](03-concurrency-control.md)). Object commands retain the + owner-shard write ownership established during execution and acquire any + additional forwarding ownership they need. 3. Choose a commit timestamp above the transaction's start/lower bound and the version, lock, and read-validation bounds returned by touched entries. 4. Validate optimistic reads and release the data-read ownership that no diff --git a/tx_service/include/cc/cc_handler.h b/tx_service/include/cc/cc_handler.h index 74e7764b9..a72008cd5 100644 --- a/tx_service/include/cc/cc_handler.h +++ b/tx_service/include/cc/cc_handler.h @@ -95,6 +95,7 @@ class CcHandler uint32_t command_id, uint64_t ts, bool is_insert, + size_t reserve_bytes, CcHandlerResult> &hres, uint32_t hd_res_idx, CcProtocol proto, diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index ec9cc363f..2f919c107 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -374,6 +374,7 @@ struct AcquireCc int64_t tx_term, uint64_t ts, bool is_insert, + size_t reserve_bytes, CcHandlerResult> *res, uint32_t hd_res_idx, CcProtocol proto, @@ -390,6 +391,8 @@ struct AcquireCc ts_ = ts; schema_version_ = schema_version; is_insert_ = is_insert; + reserve_bytes_ = reserve_bytes; + accepts_error_with_write_lock_ = true; cce_ptr_ = nullptr; hd_result_idx_ = hd_res_idx; is_local_ = true; @@ -409,7 +412,9 @@ struct AcquireCc uint32_t hd_res_idx, CcProtocol proto, IsolationLevel iso_level, - bool abort_if_oom) + bool abort_if_oom, + size_t reserve_bytes = 0, + bool accepts_error_with_write_lock = false) { uint32_t ng_id = Sharder::Instance().ShardToCcNodeGroup(key_shard_code); TemplatedCcRequest>::Reset( @@ -421,6 +426,8 @@ struct AcquireCc ts_ = ts; schema_version_ = schema_version; is_insert_ = is_insert; + reserve_bytes_ = reserve_bytes; + accepts_error_with_write_lock_ = accepts_error_with_write_lock; cce_ptr_ = nullptr; hd_result_idx_ = hd_res_idx; is_local_ = false; @@ -453,6 +460,20 @@ struct AcquireCc return is_insert_; } + size_t ReserveBytes() const + { + return reserve_bytes_; + } + + /** + * Whether the coordinator consumes acquired-lock metadata on an error. + * Required for retaining a write lock until transaction-wide OOM cleanup. + */ + bool AcceptsErrorWithWriteLock() const + { + return accepts_error_with_write_lock_; + } + uint32_t KeyShardCode() const { return key_shard_code_; @@ -500,6 +521,8 @@ struct AcquireCc uint64_t ts_; uint64_t schema_version_; bool is_insert_; + size_t reserve_bytes_{0}; + bool accepts_error_with_write_lock_{false}; // The pointer of the cc entry to which this request is directed. The // pointer is set, when the request locates the cc entry but is // blocked due to conflicts in 2PL. After the request is unblocked and diff --git a/tx_service/include/cc/cc_shard.h b/tx_service/include/cc/cc_shard.h index 6de474ea1..e133a1014 100644 --- a/tx_service/include/cc/cc_shard.h +++ b/tx_service/include/cc/cc_shard.h @@ -230,9 +230,35 @@ class CcShardHeap uint32_t SetAsDefaultArena(); - // Check if this heap is full + // Check if this heap is full. The check uses only physically allocated + // bytes; the optional output arguments also report the physical allocator + // statistics, or -1 if unavailable. Does not refresh jemalloc's epoch. bool Full(int64_t *alloc = nullptr, int64_t *commit = nullptr) const; + // Check sampled physical bytes plus outstanding payload reservations. + // Jemalloc uses EpochWorker's periodic sample, so this is a soft limit. + // Outputs remain physical-only; query failure reports -1 and returns full. + bool FullWithReservation(int64_t *alloc = nullptr, + int64_t *commit = nullptr) const; + + // Atomically reserve @p bytes against the heap limit using sampled stats. + // Returns false on query failure, ledger overflow, or insufficient + // capacity. + bool TryReserve(size_t bytes); + + // Release a reservation previously acquired with TryReserve(). It is an + // accounting error to release more bytes than are currently reserved. + void ReleaseReservation(size_t bytes); + + // Check capacity using sampled stats, without changing reservations. + // Query failure or an unrepresentable reservation returns false. + bool HasCapacityFor(size_t bytes) const; + + int64_t ReservedBytes() const + { + return reserved_bytes_.load(std::memory_order_acquire); + } + bool NeedCleanShard(int64_t &alloc, int64_t &commit) const; bool NeedDefragment(int64_t *alloc, int64_t *commit) const; @@ -271,9 +297,23 @@ class CcShardHeap } private: + /** + * Read allocator statistics without refreshing jemalloc's global epoch. + * EpochWorker supplies periodic samples; admission does not force updates. + * On query failure, return false and report -1, never zero. + */ + bool GetStats(int64_t *allocated, int64_t *committed) const; + + // True when uint64(allocated) + uint64(reserved) >= memory_limit_. + bool ExceedsLimit(int64_t allocated, int64_t reserved) const; + CcShard *cc_shard_{nullptr}; mi_heap_t *heap_{nullptr}; const size_t memory_limit_{0}; + // Bytes reserved by TryReserve() for payloads that have not been copied + // into CCEs yet. FullWithReservation() adds this to the physical + // allocated bytes before comparing with memory_limit_. + std::atomic reserved_bytes_{0}; size_t last_failed_collect_ts_{0}; // defrag heap cc for this shard @@ -414,6 +454,11 @@ class CcShard size_t WaitListSizeForMemory(); + /** + * Schedule the single cleaner from this shard's execution context only. + * InUse/Use and the scheduler's Free are serialized by that context; + * background threads must enqueue a shard request instead of calling here. + */ void WakeUpShardCleanCc(); /** diff --git a/tx_service/include/cc/local_cc_handler.h b/tx_service/include/cc/local_cc_handler.h index c60902c47..eae52bd3a 100644 --- a/tx_service/include/cc/local_cc_handler.h +++ b/tx_service/include/cc/local_cc_handler.h @@ -51,6 +51,7 @@ class LocalCcHandler : public CcHandler uint32_t command_id, uint64_t ts, bool is_insert, + size_t reserve_bytes, CcHandlerResult> &hres, uint32_t hd_res_idx, CcProtocol proto, diff --git a/tx_service/include/cc/non_blocking_lock.h b/tx_service/include/cc/non_blocking_lock.h index 82bba8989..5b5ffb1c5 100644 --- a/tx_service/include/cc/non_blocking_lock.h +++ b/tx_service/include/cc/non_blocking_lock.h @@ -87,6 +87,24 @@ class NonBlockingLock blocking_queue_.Reset(); queue_block_cmds_.Reset(); wlock_ts_ = 0; + reserved_bytes_ = 0; + } + + void SetReservedBytes(size_t bytes) + { + reserved_bytes_ = bytes; + } + + size_t TakeReservedBytes() + { + size_t bytes = reserved_bytes_; + reserved_bytes_ = 0; + return bytes; + } + + size_t ReservedBytes() const + { + return reserved_bytes_; } /** @@ -364,6 +382,10 @@ class NonBlockingLock WriteLockType write_lk_type_{WriteLockType::NoWritelock}; TxNumber write_txn_{0}; + // CcShardHeap bytes reserved on behalf of the current write-lock owner. + // Released when the write lock is released or cleared. + size_t reserved_bytes_{0}; + // The time when a write tx acquires the write lock on this lock. uint64_t wlock_ts_{0}; diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index cd962fc38..2533e6c32 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -217,6 +217,15 @@ class TemplateCcMap : public CcMap return true; } + // Older coordinators discard lock ownership on error responses. Refuse + // budgeted requests before any upgrade unless the sender can clean up + // an acquired write lock returned with OUT_OF_MEMORY. + if (req.ReserveBytes() > 0 && !req.AcceptsErrorWithWriteLock()) + { + hd_res->SetError(CcErrorCode::OUT_OF_MEMORY); + return true; + } + LockType acquired_lock = LockType::NoLock; CcErrorCode err_code = CcErrorCode::NO_ERROR; if (req.CcePtr() != nullptr) @@ -336,10 +345,13 @@ class TemplateCcMap : public CcMap if (cce_ptr == nullptr) { - // The acquire request needs a new cc entry but the cc map - // has reached the maximal capacity. - shard_->EnqueueWaitListIfMemoryFull(&req); - return false; + // Acquire is a pre-WAL stage. Even a delete/command with + // no payload budget can be part of a transaction that + // already holds reservations on other keys or shards. + // Finish this request and let the coordinator abort instead + // of adding an invisible memory-dependency edge. + hd_res->SetError(CcErrorCode::OUT_OF_MEMORY); + return true; } assert(cce_ptr != nullptr); @@ -407,6 +419,36 @@ class TemplateCcMap : public CcMap curr_version_ts = curr_version_ts > 0 ? curr_version_ts : shard_->Now(); acquire_key_result.commit_ts_ = curr_version_ts; + + // Reserve CcShardHeap bytes for the payload that PostWrite + // will install after WAL commit. This covers primary, + // secondary, and unique-secondary record writes. Shell/key/page + // allocations are sampled separately by the heap's physical + // statistics; this charge estimates only the source payload. + if (req.ReserveBytes() > 0 && + cc_entry.GetKeyLock()->ReservedBytes() == 0) + { + CcShardHeap *heap = shard_->GetShardHeap(); + if (heap != nullptr && heap->TryReserve(req.ReserveBytes())) + { + cc_entry.GetKeyLock()->SetReservedBytes( + req.ReserveBytes()); + } + else + { + // An upgrade may have consumed a read lock/intent + // whose address is still in the coordinator's read set. + // Keep the acquired lock and publish its result even on + // OOM. AggregateAcquiredKeys transfers ownership to the + // write set, and the normal abort tail releases it. + // Never wait here: other keys/shards may already hold + // reservations needed to finish this acquire stage. + shard_->WakeUpShardCleanCc(); + hd_res->SetError(CcErrorCode::OUT_OF_MEMORY); + return true; + } + } + if (cce_ptr->PayloadStatus() == RecordStatus::Unknown) { if (txservice_enable_key_cache && table_name_.IsBase()) @@ -9518,10 +9560,9 @@ class TemplateCcMap : public CcMap } if (shard_heap != nullptr && - shard_heap->Full(&heap_alloc, &heap_commit)) + shard_heap->FullWithReservation(&heap_alloc, &heap_commit)) { - if (txservice_enable_cache_replacement && - shard_heap->NeedCleanShard(heap_alloc, heap_commit)) + if (txservice_enable_cache_replacement) { // The shard has reached the maximal capacity. Tries to clean cc // entries that have been checkpointed but are not being @@ -9835,10 +9876,9 @@ class TemplateCcMap : public CcMap } if (shard_heap != nullptr && - shard_heap->Full(&heap_alloc, &heap_commit)) + shard_heap->FullWithReservation(&heap_alloc, &heap_commit)) { - if (txservice_enable_cache_replacement && - shard_heap->NeedCleanShard(heap_alloc, heap_commit)) + if (txservice_enable_cache_replacement) { // The shard has reached the maximal capacity. Tries to clean cc // entries that have been checkpointed but are not being diff --git a/tx_service/include/proto/cc_request.proto b/tx_service/include/proto/cc_request.proto index e0718e811..1d45794be 100644 --- a/tx_service/include/proto/cc_request.proto +++ b/tx_service/include/proto/cc_request.proto @@ -819,11 +819,20 @@ message AcquireRequest { IsolationType iso_level = 11; uint64 schema_version = 12; bool abort_if_oom = 13; + // Bytes to reserve in the destination CcShardHeap for the payload that + // will be installed by the matching PostWrite request. + uint64 reserve_bytes = 14; + // The coordinator consumes cce_addr/commit_ts even on a final error. + // Without this capability, budgeted acquires are rejected before locking. + bool accepts_error_with_write_lock = 15; } message AcquireResponse { int32 error_code = 1; uint64 vali_ts = 2; + // A nonzero commit_ts on a final result transfers write-lock ownership, + // including OUT_OF_MEMORY, when the request advertised that capability. + // An ACK carries only an address, not an acquired write lock. uint64 commit_ts = 3; CceAddr_msg cce_addr = 4; // The response is an acknowledgement or the result diff --git a/tx_service/include/read_write_set.h b/tx_service/include/read_write_set.h index 01254bc4c..fecf1c0e3 100644 --- a/tx_service/include/read_write_set.h +++ b/tx_service/include/read_write_set.h @@ -296,6 +296,10 @@ class ReadWriteSet OperationType op_type, bool check_unique = false) { + if (op_type == OperationType::Delete) + { + rec.reset(); + } // Check write set bytes count. wset_bytes_cnt_ += ((tx_key.KeyPtr() ? tx_key.SerializedLength() : 0) + (rec.get() ? rec.get()->SerializedLength() : 0)); diff --git a/tx_service/include/remote/remote_cc_handler.h b/tx_service/include/remote/remote_cc_handler.h index 90bd060d8..c8bcf43c3 100644 --- a/tx_service/include/remote/remote_cc_handler.h +++ b/tx_service/include/remote/remote_cc_handler.h @@ -56,6 +56,7 @@ class RemoteCcHandler uint32_t command_id, uint64_t ts, bool is_insert, + size_t reserve_bytes, CcHandlerResult> &hres, uint32_t hd_res_idx, CcProtocol proto = CcProtocol::OCC, diff --git a/tx_service/include/util.h b/tx_service/include/util.h index abeb7afcb..73c1e9237 100644 --- a/tx_service/include/util.h +++ b/tx_service/include/util.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -451,29 +452,40 @@ static inline bool IsStandbyTx(int64_t tx_term) } #if defined(WITH_JEMALLOC) -static inline void GetJemallocArenaStat(uint32_t arena_id, +/** + * Read the current arena snapshot without refreshing the global epoch. Return + * false on query failure or overflow and leave outputs unchanged. + */ +static inline bool GetJemallocArenaStat(uint32_t arena_id, size_t &resident, size_t &allocated) { - // estimate thread memory usage from total process memory size_t total_resident = 0; size_t small_allocated = 0; size_t large_allocated = 0; size_t sz = sizeof(total_resident); - // uint64_t epoch = 1; - // mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)); - char mib[64]; snprintf(mib, sizeof(mib), "stats.arenas.%u.resident", arena_id); - mallctl(mib, &total_resident, &sz, NULL, 0); + if (mallctl(mib, &total_resident, &sz, nullptr, 0) != 0) + { + return false; + } snprintf(mib, sizeof(mib), "stats.arenas.%u.small.allocated", arena_id); - mallctl(mib, &small_allocated, &sz, NULL, 0); + if (mallctl(mib, &small_allocated, &sz, nullptr, 0) != 0) + { + return false; + } snprintf(mib, sizeof(mib), "stats.arenas.%u.large.allocated", arena_id); - mallctl(mib, &large_allocated, &sz, NULL, 0); + if (mallctl(mib, &large_allocated, &sz, nullptr, 0) != 0 || + small_allocated > std::numeric_limits::max() - large_allocated) + { + return false; + } resident = total_resident; allocated = small_allocated + large_allocated; + return true; } #endif // TSC frequency in cycles per microsecond (measured at initialization) diff --git a/tx_service/src/cc/cc_entry.cpp b/tx_service/src/cc/cc_entry.cpp index 96fc5cf30..a029af992 100644 --- a/tx_service/src/cc/cc_entry.cpp +++ b/tx_service/src/cc/cc_entry.cpp @@ -168,6 +168,18 @@ void LruEntry::ClearLocks(CcShard &ccs, NonBlockingLock *key_lock = cc_lock_and_extra_->KeyLock(); + // Release any CcShardHeap reservation still attached to this lock before + // the lock object is recycled. + const size_t reserved = key_lock->TakeReservedBytes(); + if (reserved != 0) + { + CcShardHeap *heap = ccs.GetShardHeap(); + if (heap != nullptr) + { + heap->ReleaseReservation(reserved); + } + } + // Deletes the write lock/intent. auto [w_tx, w_type] = key_lock->WriteTx(); if (w_type != NonBlockingLock::WriteLockType::NoWritelock) diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 19a0c298e..55551c85e 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -1669,102 +1669,77 @@ void RestoreCcMapCc::DecodedDataItem( bool ShardCleanCc::Execute(CcShard &ccs) { CcShardHeap *shard_heap = ccs.GetShardHeap(); - int64_t heap_alloc, heap_commit; - if (shard_heap != nullptr && shard_heap->Full(&heap_alloc, &heap_commit)) + if (shard_heap != nullptr && shard_heap->FullWithReservation()) { assert(txservice_enable_cache_replacement); - bool need_yield = false; - if (shard_heap->NeedCleanShard(heap_alloc, heap_commit)) - { - size_t free_size = 0; - std::tie(free_size, need_yield) = ccs.Clean(); - free_count_ += free_size; - } + // Reservations can exhaust admission capacity below the physical + // cleaning watermark. Scan clean entries in that case too. + auto [free_size, need_yield] = ccs.Clean(); + free_count_ += free_size; - if (shard_heap->Full(&heap_alloc, &heap_commit) && - shard_heap->NeedCleanShard(heap_alloc, heap_commit)) + if (shard_heap->FullWithReservation()) { if (need_yield) { - // Continue to clean in the next run one round. ccs.Enqueue(this); return false; } - else - { - // Reached the tail ccpage with the allocated memory still - // above the heap threshold. Abort only the parked requests - // that opted in via AbortIfOom() -- txs holding range read - // locks, which would deadlock reclaim by blocking the very - // data sync that frees memory. Ordinary requests stay parked - // until eviction or a flush makes room. - ccs.AbortRequestsAfterMemoryFree(); - - // Notify the checkpointer thread to do checkpoint if there - // is not freeable entries to be kicked out from ccmap and - // if the shard is not doing defrag. - if (free_count_ == 0 && !shard_heap->IsDefragHeapCcOnFly() && - !Sharder::Instance().GetCheckpointer()->IsOngoingDataSync()) - { - ccs.NotifyCkpt(true, "memory_pressure"); - } - - free_count_ = 0; - // This pass decided there was nothing to free from state it - // gathered before any wake-up that arrived while it ran -- a - // checkpoint flush turning dirty entries evictable, say. Such - // a wake-up could not be delivered because this request was in - // use, so re-run rather than stopping with requests parked and - // nothing left to schedule another pass. - if (ccs.TakeShardCleanCcWakeUp() && - ccs.WaitListSizeForMemory() > 0) - { - ccs.Enqueue(this); - return false; - } - - // Return true will set the request as free, which means the - // request is not in working state. - return true; + // Reached the tail ccpage with the effective heap still above the + // threshold. Abort only the parked requests that opted in via + // AbortIfOom() -- txs holding range read locks, which would + // deadlock reclaim by blocking the very data sync that frees + // memory. Ordinary requests stay parked until eviction or a flush + // makes room. + ccs.AbortRequestsAfterMemoryFree(); + + // Notify the checkpointer thread to do checkpoint if there are no + // freeable entries to be kicked out from ccmap and if the shard is + // not doing defrag. + if (free_count_ == 0 && !shard_heap->IsDefragHeapCcOnFly() && + !Sharder::Instance().GetCheckpointer()->IsOngoingDataSync()) + { + // This only helps future attempts. Dispatch completion is not + // a flush barrier, and must not control transaction completion. + ccs.NotifyCkpt(true, "memory_pressure"); } - } - else - { - // Get the free memory, re-run a batch of the waiting ccrequest. - bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); - if (!wait_list_empty) + free_count_ = 0; + + // This pass decided there was nothing to free from state it + // gathered before any wake-up that arrived while it ran -- a + // checkpoint flush turning dirty entries evictable, say. Such a + // wake-up could not be delivered because this request was in use, + // so re-run rather than stopping with requests parked and nothing + // left to schedule another pass. + if (ccs.TakeShardCleanCcWakeUp() && + ccs.WaitListSizeForMemory() > 0) { ccs.Enqueue(this); + return false; } - // Reset the value if the ccrequest is finished. - free_count_ = (wait_list_empty) ? 0 : free_count_; - if (wait_list_empty) - { - // Nothing is parked any more, so a wake-up that arrived during - // this pass has already been served. - ccs.TakeShardCleanCcWakeUp(); - } - return wait_list_empty; + // Return true will set the request as free, which means the + // request is not in working state. + return true; } } + + // There is available memory on this shard, re-run a batch of the waiting + // ccrequest if has any waiting request, otherwise, finish this shard clean + // ccrequests. + bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); + if (!wait_list_empty) + { + ccs.Enqueue(this); + } else { - // There is available memory on this shard, re-run a batch of the - // waiting ccrequest if has any waiting request, otherwise, finish - // this shard clean ccrequests. - bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); - if (!wait_list_empty) - { - ccs.Enqueue(this); - } - else - { - ccs.TakeShardCleanCcWakeUp(); - } - return wait_list_empty; + free_count_ = 0; + // Nothing is parked any more, so a wake-up that arrived during this + // pass has already been served. + ccs.TakeShardCleanCcWakeUp(); } + return wait_list_empty; } void FetchTableRangeSizeCc::Reset(const TableName &table_name, diff --git a/tx_service/src/cc/cc_shard.cpp b/tx_service/src/cc/cc_shard.cpp index 9a834889f..6e1e476f3 100644 --- a/tx_service/src/cc/cc_shard.cpp +++ b/tx_service/src/cc/cc_shard.cpp @@ -29,6 +29,7 @@ #include // std::chrono #include #include // std::setprecision +#include #include #include "cc/catalog_cc_map.h" @@ -974,6 +975,15 @@ KeyGapLockAndExtraData *CcShard::NewLock(CcMap *ccm, KeyGapLockAndExtraData *lk = lock_vec_.at(next_lock_idx_).get(); assert(!lk->GetUsedStatus()); + const size_t reserved = lk->KeyLock()->TakeReservedBytes(); + if (reserved != 0) + { + CcShardHeap *heap = GetShardHeap(); + if (heap != nullptr) + { + heap->ReleaseReservation(reserved); + } + } lk->Reset(ccm, page, entry); lk->SetUsedStatus(true); used_lock_count_++; @@ -3206,29 +3216,90 @@ uint32_t CcShardHeap::SetAsDefaultArena() #endif } -bool CcShardHeap::Full(int64_t *alloc, int64_t *commit) const +bool CcShardHeap::GetStats(int64_t *allocated, int64_t *committed) const { + *allocated = -1; + *committed = -1; #if defined(WITH_JEMALLOC) - // estimate thread memory usage from total process memory + // EpochWorker refreshes global statistics periodically. Keep this hot path + // read-only to avoid per-request global statistics refresh work. + // The limit is soft: installed bytes and tcache activity can be absent from + // the current sample even after their payload reservations are released. size_t total_resident = 0; size_t total_allocated = 0; - GetJemallocArenaStat(arena_id_, total_resident, total_allocated); + if (!GetJemallocArenaStat(arena_id_, total_resident, total_allocated) || + total_allocated > + static_cast(std::numeric_limits::max()) || + total_resident > + static_cast(std::numeric_limits::max())) + { + return false; + } + + *allocated = static_cast(total_allocated); + *committed = static_cast(total_resident); +#else + mi_thread_stats(allocated, committed); +#endif + if (*allocated < 0 || *committed < 0) + { + *allocated = -1; + *committed = -1; + return false; + } + return true; +} + +bool CcShardHeap::ExceedsLimit(int64_t allocated, int64_t reserved) const +{ + if (allocated < 0 || reserved < 0) + { + // A negative value means the accounting has already underflowed. + // Treat the heap as full so callers stop admitting new work. + return true; + } + + // Two nonnegative int64_t values sum without overflow in uint64_t. + uint64_t used = static_cast(allocated) + + static_cast(reserved); + return used >= static_cast(memory_limit_); +} + +bool CcShardHeap::Full(int64_t *alloc, int64_t *commit) const +{ + int64_t allocated = 0; + int64_t committed = 0; + const bool valid_stats = GetStats(&allocated, &committed); if (alloc != nullptr) { - *alloc = total_allocated; + *alloc = allocated; } if (commit != nullptr) { - *commit = total_resident; + *commit = committed; } - return total_allocated >= memory_limit_; + if (!valid_stats || ExceedsLimit(allocated, 0)) + { + return true; + } +#if defined(WITH_JEMALLOC) + return false; #else - int64_t allocated, committed; - mi_thread_stats(&allocated, &committed); + return cc_shard_->EnableDefragment() && + static_cast(committed) > memory_limit_; +#endif +} + +bool CcShardHeap::FullWithReservation(int64_t *alloc, int64_t *commit) const +{ + int64_t allocated = 0; + int64_t committed = 0; + const bool valid_stats = GetStats(&allocated, &committed); + if (alloc != nullptr) { *alloc = allocated; @@ -3239,12 +3310,152 @@ bool CcShardHeap::Full(int64_t *alloc, int64_t *commit) const *commit = committed; } - return allocated >= static_cast(memory_limit_) || - (cc_shard_->EnableDefragment() && - committed > static_cast(memory_limit_)); + if (!valid_stats || + ExceedsLimit(allocated, + reserved_bytes_.load(std::memory_order_acquire))) + { + return true; + } + +#if defined(WITH_JEMALLOC) + return false; +#else + return cc_shard_->EnableDefragment() && + static_cast(committed) > memory_limit_; #endif } +bool CcShardHeap::TryReserve(size_t bytes) +{ + if (bytes == 0) + { + return true; + } + + if (bytes > memory_limit_ || + bytes > static_cast(std::numeric_limits::max())) + { + return false; + } + + const uint64_t limit = static_cast(memory_limit_); + const uint64_t requested = static_cast(bytes); + int64_t reserved = reserved_bytes_.load(std::memory_order_relaxed); + + for (;;) + { + int64_t allocated = 0; + int64_t committed = 0; + if (!GetStats(&allocated, &committed)) + { + return false; + } + + if (reserved < 0 || + requested > static_cast( + std::numeric_limits::max() - reserved)) + { + // Refuse a corrupt ledger or an unrepresentable reservation sum + // before converting bytes or adding to the signed ledger. + return false; + } + +#if defined(WITH_JEMALLOC) + (void) committed; +#else + if (cc_shard_->EnableDefragment() && + static_cast(committed) > memory_limit_) + { + return false; + } +#endif + + const uint64_t used = static_cast(allocated) + + static_cast(reserved); + if (used >= limit || requested > limit - used) + { + return false; + } + + if (reserved_bytes_.compare_exchange_weak( + reserved, + reserved + static_cast(bytes), + std::memory_order_acq_rel, + std::memory_order_relaxed)) + { + return true; + } + } +} + +void CcShardHeap::ReleaseReservation(size_t bytes) +{ + if (bytes == 0) + { + return; + } + + if (bytes > memory_limit_ || + bytes > static_cast(std::numeric_limits::max())) + { + LOG(ERROR) << "CcShardHeap invalid reservation release: " + << "release=" << bytes << ", limit=" << memory_limit_; + return; + } + + const int64_t delta = static_cast(bytes); + int64_t reserved = reserved_bytes_.load(std::memory_order_relaxed); + while (reserved >= delta) + { + if (reserved_bytes_.compare_exchange_weak( + reserved, + reserved - delta, + std::memory_order_acq_rel, + std::memory_order_relaxed)) + { + return; + } + } + + LOG(ERROR) << "CcShardHeap reservation underflow: reserved=" << reserved + << ", release=" << bytes << ", limit=" << memory_limit_; +} + +bool CcShardHeap::HasCapacityFor(size_t bytes) const +{ + const uint64_t limit = static_cast(memory_limit_); + const uint64_t requested = static_cast(bytes); + + int64_t allocated = 0; + int64_t committed = 0; + if (!GetStats(&allocated, &committed)) + { + return false; + } + int64_t reserved = reserved_bytes_.load(std::memory_order_acquire); + + if (reserved < 0 || + requested > static_cast(std::numeric_limits::max() - + reserved)) + { + return false; + } + +#if defined(WITH_JEMALLOC) + (void) committed; +#else + if (cc_shard_->EnableDefragment() && + static_cast(committed) > memory_limit_) + { + return false; + } +#endif + + const uint64_t used = static_cast(allocated) + + static_cast(reserved); + return used < limit && requested <= limit - used; +} + bool CcShardHeap::NeedCleanShard(int64_t &alloc, int64_t &commit) const { return alloc >= static_cast(memory_limit_) || diff --git a/tx_service/src/cc/local_cc_handler.cpp b/tx_service/src/cc/local_cc_handler.cpp index 674f719b1..d2f2d62fe 100644 --- a/tx_service/src/cc/local_cc_handler.cpp +++ b/tx_service/src/cc/local_cc_handler.cpp @@ -58,6 +58,7 @@ void txservice::LocalCcHandler::AcquireWrite( uint32_t command_id, uint64_t ts, bool is_insert, + size_t reserve_bytes, CcHandlerResult> &hres, uint32_t hd_res_idx, CcProtocol proto, @@ -88,6 +89,7 @@ void txservice::LocalCcHandler::AcquireWrite( tx_term, ts, is_insert, + reserve_bytes, &hres, hd_res_idx, proto, @@ -114,6 +116,7 @@ void txservice::LocalCcHandler::AcquireWrite( command_id, ts, is_insert, + reserve_bytes, hres, hd_res_idx, proto, diff --git a/tx_service/src/cc/non_blocking_lock.cpp b/tx_service/src/cc/non_blocking_lock.cpp index b9d09311d..86279e2bb 100644 --- a/tx_service/src/cc/non_blocking_lock.cpp +++ b/tx_service/src/cc/non_blocking_lock.cpp @@ -397,6 +397,18 @@ bool NonBlockingLock::ReleaseWriteLock(TxNumber tx_number, return false; } + const size_t reserved = TakeReservedBytes(); + if (reserved != 0 && ccs != nullptr) + { + CcShardHeap *heap = ccs->GetShardHeap(); + if (heap != nullptr) + { + heap->ReleaseReservation(reserved); + // Re-check memory waiters now that reserved capacity is free. + ccs->WakeUpShardCleanCc(); + } + } + write_lk_type_ = WriteLockType::NoWritelock; write_txn_ = 0; @@ -492,6 +504,18 @@ bool NonBlockingLock::ReleaseWriteIntent(TxNumber tx_number, CcShard *ccs) return false; } + const size_t reserved = TakeReservedBytes(); + if (reserved != 0 && ccs != nullptr) + { + CcShardHeap *heap = ccs->GetShardHeap(); + if (heap != nullptr) + { + heap->ReleaseReservation(reserved); + // Re-check memory waiters now that reserved capacity is free. + ccs->WakeUpShardCleanCc(); + } + } + write_lk_type_ = WriteLockType::NoWritelock; write_txn_ = 0; @@ -595,6 +619,19 @@ LockType NonBlockingLock::ClearTx(TxNumber tx_number, ? LockType::WriteLock : LockType::WriteIntent; + const size_t reserved = TakeReservedBytes(); + if (reserved != 0 && ccs != nullptr) + { + CcShardHeap *heap = ccs->GetShardHeap(); + if (heap != nullptr) + { + heap->ReleaseReservation(reserved); + // Re-check memory waiters now that reserved capacity is + // free. + ccs->WakeUpShardCleanCc(); + } + } + write_lk_type_ = WriteLockType::NoWritelock; write_txn_ = 0; if (object != nullptr && return_type == LockType::WriteLock && diff --git a/tx_service/src/remote/cc_stream_receiver.cpp b/tx_service/src/remote/cc_stream_receiver.cpp index d7aa97829..4b3d1c960 100644 --- a/tx_service/src/remote/cc_stream_receiver.cpp +++ b/tx_service/src/remote/cc_stream_receiver.cpp @@ -523,11 +523,24 @@ void CcStreamReceiver::OnReceiveCcMsg(std::unique_ptr msg) if (cc_res.error_code() != 0) { - if (acq_res.cce_addr_.Term() < 0) + bool first_response = acq_res.cce_addr_.Term() < 0; + if (!cc_res.is_ack() && cc_res.commit_ts() != 0 && + cce_addr_res.cce_lock_ptr() != 0 && cce_addr_res.term() >= 0) { - acq_res.remote_ack_cnt_->fetch_sub(1); + // Publish ownership before completion, including ACK -> OOM. + // AggregateAcquiredKeys must include this lock in abort + // cleanup. + acq_res.cce_addr_.SetCceLock(cce_addr_res.cce_lock_ptr(), + cce_addr_res.term(), + cce_addr_res.core_id()); + acq_res.last_vali_ts_ = cc_res.vali_ts(); + acq_res.commit_ts_ = cc_res.commit_ts(); + } + if (first_response) + { + acq_res.remote_ack_cnt_->fetch_sub(1, + std::memory_order_release); } - acq_res.SetRemoteHdResult(true, std::memory_order_release); hd_res->SetRemoteError( ToLocalType::ConvertCcErrorCode(cc_res.error_code())); @@ -1670,26 +1683,19 @@ void CcStreamReceiver::OnReceiveCcMsg(std::unique_ptr msg) break; } - acq_key_result_vec.at(resp.acq_key_result_vec_idx()) - .SetRemoteHdResult(true, std::memory_order_release); - AckStatus status = (AckStatus) resp.req_status(); - if (status == AckStatus::ErrorTerm) - { - hd_res->SetError(CcErrorCode::NG_TERM_CHANGED); - } - else if (status == AckStatus::Finished) - { - // The cc request has been executed or aborted on the remote - // node, the result is unknown. Treat it as the response is - // lost otherwise txm might block forever if the response is - // really lost. - hd_res->SetError(CcErrorCode::REQUEST_LOST); - } - else if (status == AckStatus::BlockQueue) + if (status == AckStatus::ErrorTerm || + status == AckStatus::Finished) { - // The cc request is still in blocking queue. The txm - // should keep waiting. + // Only a terminal probe consumes the remote request reference. + // A BlockQueue probe is not the acquire result: publishing the + // flag for it would discard the eventual success/OOM response + // and prevent transaction-wide abort from ever running. + acq_key_result_vec.at(resp.acq_key_result_vec_idx()) + .SetRemoteHdResult(true, std::memory_order_release); + hd_res->SetRemoteError(status == AckStatus::ErrorTerm + ? CcErrorCode::NG_TERM_CHANGED + : CcErrorCode::REQUEST_LOST); } hd_res->UnsetBlockReqCheckTs(); diff --git a/tx_service/src/remote/remote_cc_handler.cpp b/tx_service/src/remote/remote_cc_handler.cpp index c7a14cb0d..2e2f78775 100644 --- a/tx_service/src/remote/remote_cc_handler.cpp +++ b/tx_service/src/remote/remote_cc_handler.cpp @@ -46,6 +46,7 @@ void txservice::remote::RemoteCcHandler::AcquireWrite( uint32_t command_id, uint64_t ts, bool is_insert, + size_t reserve_bytes, CcHandlerResult> &hres, uint32_t hd_res_idx, CcProtocol proto, @@ -86,6 +87,8 @@ void txservice::remote::RemoteCcHandler::AcquireWrite( acq->set_ts(ts); acq->set_schema_version(schema_version); acq->set_insert(is_insert); + acq->set_reserve_bytes(reserve_bytes); + acq->set_accepts_error_with_write_lock(true); acq->set_key_shard_code(key_shard_code); acq->set_protocol(ToRemoteType::ConvertProtocol(proto)); acq->set_iso_level(ToRemoteType::ConvertIsolation(iso_level)); diff --git a/tx_service/src/remote/remote_cc_request.cpp b/tx_service/src/remote/remote_cc_request.cpp index 8c9af2f9f..bc8715315 100644 --- a/tx_service/src/remote/remote_cc_request.cpp +++ b/tx_service/src/remote/remote_cc_request.cpp @@ -70,7 +70,10 @@ txservice::remote::RemoteAcquire::RemoteAcquire() ToRemoteType::ConvertCcErrorCode(res->ErrorCode())); resp->set_vec_idx(acquire_req.vec_idx()); - if (!cc_res_.IsError()) + // A nonzero commit_ts denotes an acquired write lock, not merely the + // address sent in a blocked-request ACK. Preserve that ownership on OOM + // so the coordinator's abort tail can release it. + if (!cc_res_.IsError() || cc_res_.Value()[0].commit_ts_ != 0) { const AcquireKeyResult &acquire_key_res = cc_res_.Value()[0]; @@ -127,7 +130,9 @@ void txservice::remote::RemoteAcquire::Reset( req.vec_idx(), ToLocalType::ConvertProtocol(req.protocol()), ToLocalType::ConvertIsolation(req.iso_level()), - req.abort_if_oom()); + req.abort_if_oom(), + req.reserve_bytes(), + req.accepts_error_with_write_lock()); input_msg_ = std::move(input_msg); diff --git a/tx_service/src/tx_execution.cpp b/tx_service/src/tx_execution.cpp index 01e337b75..f41816f2d 100644 --- a/tx_service/src/tx_execution.cpp +++ b/tx_service/src/tx_execution.cpp @@ -3928,6 +3928,9 @@ void TransactionExecution::Process(AcquireWriteOperation &acquire_write) { acquire_write.acquire_write_entries_[entry_idx++] = &write_entry; + const size_t reserve_bytes = + write_entry.rec_ ? write_entry.rec_->MemUsage() : 0; + // TODO: enable is_insert after Serializable Isolation is // supported. cc_handler_->AcquireWrite( @@ -3940,6 +3943,7 @@ void TransactionExecution::Process(AcquireWriteOperation &acquire_write) command_id_.load(std::memory_order_relaxed), current_ts, false, + reserve_bytes, acquire_write.hd_result_, res_idx++, protocol_, @@ -3958,6 +3962,7 @@ void TransactionExecution::Process(AcquireWriteOperation &acquire_write) command_id_.load(std::memory_order_relaxed), current_ts, false, + reserve_bytes, acquire_write.hd_result_, res_idx++, protocol_, @@ -7280,6 +7285,7 @@ void TransactionExecution::Process(CmdForwardAcquireWriteOp &forward_acquire) command_id_.load(std::memory_order_relaxed), current_ts, false, + 0, forward_acquire.hd_result_, res_idx++, protocol_, diff --git a/tx_service/tests/CMakeLists.txt b/tx_service/tests/CMakeLists.txt index 41fed6301..e21d7879c 100644 --- a/tx_service/tests/CMakeLists.txt +++ b/tx_service/tests/CMakeLists.txt @@ -121,6 +121,15 @@ foreach(name ${CATCH_MAIN_TESTS}) add_substrate_test(${name} Catch2::Catch2WithMain) endforeach() +# Exercise real read/acquire/abort/retry on TestNode; bound old memory waits. +add_executable(PayloadReservation-Test ${TESTS_DIR}/PayloadReservation-Test.cpp) +target_link_libraries(PayloadReservation-Test PRIVATE + test_harness Catch2::Catch2WithMain) +set_property(TARGET PayloadReservation-Test PROPERTY CXX_STANDARD 20) +set_property(TARGET PayloadReservation-Test PROPERTY CXX_EXTENSIONS OFF) +catch_discover_tests(PayloadReservation-Test PROPERTIES + LABELS "PayloadReservation-Test" TIMEOUT 60) + foreach(name ${OWN_MAIN_TESTS}) add_substrate_test(${name} Catch2::Catch2) endforeach() diff --git a/tx_service/tests/PayloadReservation-Test.cpp b/tx_service/tests/PayloadReservation-Test.cpp new file mode 100644 index 000000000..f72f2032b --- /dev/null +++ b/tx_service/tests/PayloadReservation-Test.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include +#include +#include + +#include "catch2/catch_all.hpp" +#include "cc/cc_req_misc.h" +#include "cc/cc_shard.h" +#include "cc/local_cc_shards.h" +#include "error_messages.h" +#include "harness/test_node.h" +#include "sharder.h" +#include "tx_execution.h" +#include "tx_record.h" +#include "tx_request.h" + +using namespace txservice; +using namespace txservice::test; + +namespace +{ +/** + * Inject a declared payload footprint without allocating a heap-sized buffer. + * The real coordinator, AcquireCc, aggregation and abort tail still run. This + * controls admission only; it is not an allocator-peak or durable-WAL test. + */ +class BudgetedRecord : public CompositeRecord +{ +public: + BudgetedRecord(int value, size_t budget) + : CompositeRecord(std::move(value)), budget_(budget) + { + } + + size_t MemUsage() const override + { + return budget_; + } + + TxRecord::Uptr Clone() const override + { + return std::make_unique(*this); + } + +private: + size_t budget_; +}; + +struct HeapSnapshot +{ + size_t limit{0}; + int64_t reserved{0}; + bool full{true}; +}; + +std::vector SnapshotHeaps() +{ + LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); + std::vector result(shards->Count()); + for (size_t core = 0; core < result.size(); ++core) + { + WaitableCc snapshot( + [&, core](CcShard &ccs) + { + CcShardHeap *heap = ccs.GetShardHeap(); + if (heap != nullptr) + { + result[core] = {heap->MemoryLimit(), + heap->ReservedBytes(), + heap->FullWithReservation()}; + } + return true; + }); + shards->EnqueueCcRequest(core, &snapshot); + snapshot.Wait(); + REQUIRE_FALSE(snapshot.IsError()); + REQUIRE(result[core].limit > 0); + } + return result; +} +} // namespace + +TEST_CASE("payload OOM aborts a complete transaction before retry", + "[tx][payload]") +{ + // Sharder is process-global: construct exactly one node, without SECTIONs. + TestNode node(TestNodeOptions{}.CoreNum(2)); + const auto baseline = SnapshotHeaps(); + REQUIRE(baseline[0].limit == baseline[1].limit); + + struct ReadMode + { + CcProtocol protocol; + bool for_write; + }; + const std::array modes{{{CcProtocol::OCC, false}, + {CcProtocol::Locking, false}, + {CcProtocol::Locking, true}}}; + int key = 100; + for (const ReadMode &mode : modes) + { + auto seed = node.BeginTx(); + REQUIRE(seed.Upsert(key, 10)); + REQUIRE(seed.Upsert(key + 1, 20)); + REQUIRE(seed.Commit()); + + auto tx = node.BeginTx(IsolationLevel::RepeatableRead, mode.protocol); + TxKey read_key = Key(key); + CompositeRecord old_value; + ReadTxRequest read(&node.Table(), + node.SchemaVersion(), + &read_key, + &old_value, + mode.for_write); + tx.Txm()->Execute(&read); + read.Wait(); + REQUIRE_FALSE(read.IsError()); + REQUIRE(read.Result().first == RecordStatus::Normal); + + // This request is <= the heap limit, but cannot fit alongside existing + // allocations. It must fail at reservation, not enter the memory queue. + for (const HeapSnapshot &heap : SnapshotHeaps()) + { + REQUIRE_FALSE(heap.full); + } + UpsertTxRequest large( + &node.Table(), + Key(key), + std::make_unique(11, baseline[0].limit), + OperationType::Upsert); + tx.Txm()->Execute(&large); + large.Wait(); + REQUIRE_FALSE(large.IsError()); + REQUIRE(tx.Upsert(key + 1, 21)); + CommitTxRequest commit; + tx.Txm()->Execute(&commit); + commit.Wait(); + // A failed commit finishes its result with Result() == false and the + // error code, but not with the TxResultStatus::Error state. + REQUIRE_FALSE(commit.Result()); + REQUIRE(commit.ErrorCode() == TxErrorCode::OUT_OF_MEMORY); + + const auto after_abort = SnapshotHeaps(); + for (size_t core = 0; core < baseline.size(); ++core) + { + REQUIRE(after_abort[core].reserved == baseline[core].reserved); + } + auto verify_abort = node.BeginTx(); + int value = 0; + REQUIRE(verify_abort.Read(key, value)); + REQUIRE(value == 10); + REQUIRE(verify_abort.Read(key + 1, value)); + REQUIRE(value == 20); + REQUIRE(verify_abort.Commit()); + + // Retry the entire transaction, not the failed participant request. + auto retry = + node.BeginTx(IsolationLevel::RepeatableRead, mode.protocol); + REQUIRE(retry.Read(key, value)); + REQUIRE(retry.Upsert(key, 11)); + REQUIRE(retry.Upsert(key + 1, 21)); + REQUIRE(retry.Commit()); + auto verify_commit = node.BeginTx(); + REQUIRE(verify_commit.Read(key, value)); + REQUIRE(value == 11); + REQUIRE(verify_commit.Read(key + 1, value)); + REQUIRE(value == 21); + REQUIRE(verify_commit.Commit()); + const auto after_commit = SnapshotHeaps(); + for (size_t core = 0; core < baseline.size(); ++core) + { + REQUIRE(after_commit[core].reserved == baseline[core].reserved); + } + key += 2; + } +}