From 13526445b80f7d378967fa8abd234631cecd37be Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 1 Oct 2026 17:59:35 +0300 Subject: [PATCH 01/26] implement new way to use task allocator --- common/common-tests.cmake | 3 +- common/containers/object-pool-test.cpp | 293 ++++++++++++++++++ common/containers/object-pool.h | 109 +++++++ compiler/code-gen/vertex-compiler.cpp | 11 +- .../memory-resource/chunk-pool-resource.h | 108 +++++++ .../segmented-stack-resource.h | 117 +++++++ .../components/confdata/confdata.cmake | 4 + .../confdata/state/component-state.h | 24 +- .../confdata/state/instance-state.h | 13 +- .../components/kphp/state/component-state.cpp | 44 +++ .../components/kphp/state/component-state.h | 32 +- .../components/kphp/state/instance-state.h | 8 +- runtime-light/coroutine/coroutine-state.h | 18 +- runtime-light/coroutine/coroutine.cmake | 3 + .../detail/allocator/coroutine-allocator.h | 6 +- .../allocator/coroutine-malloc-interface.h | 14 +- .../allocator/runtime-coroutine-allocator.cpp | 15 + .../allocator/runtime-coroutine-allocator.h | 44 +++ .../detail/allocator/task-allocator.cpp | 15 + .../detail/allocator/task-allocator.h | 182 +++++++++++ .../detail/allocator/task-malloc-interface.h | 122 ++++++++ runtime-light/coroutine/shared-task.h | 4 + runtime-light/coroutine/task.h | 25 +- runtime-light/coroutine/type-traits.h | 14 +- runtime-light/runtime-light.cmake | 1 + runtime-light/stdlib/fork/fork-functions.h | 16 + .../chunk-pool-resource-test.cpp | 205 ++++++++++++ .../segmented-stack-resource-test.cpp | 289 +++++++++++++++++ tests/cpp/runtime/runtime-tests.cmake | 2 + 29 files changed, 1683 insertions(+), 58 deletions(-) create mode 100644 common/containers/object-pool-test.cpp create mode 100644 common/containers/object-pool.h create mode 100644 runtime-common/core/memory-resource/chunk-pool-resource.h create mode 100644 runtime-common/core/memory-resource/segmented-stack-resource.h create mode 100644 runtime-light/coroutine/coroutine.cmake create mode 100644 runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp create mode 100644 runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h create mode 100644 runtime-light/coroutine/detail/allocator/task-allocator.cpp create mode 100644 runtime-light/coroutine/detail/allocator/task-allocator.h create mode 100644 runtime-light/coroutine/detail/allocator/task-malloc-interface.h create mode 100644 tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp create mode 100644 tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp diff --git a/common/common-tests.cmake b/common/common-tests.cmake index 5fd0b42b23..1a0a35f7e8 100644 --- a/common/common-tests.cmake +++ b/common/common-tests.cmake @@ -18,7 +18,8 @@ prepend(COMMON_TESTS_SOURCES ${COMMON_DIR}/ wrappers/span-test.cpp wrappers/string_view-test.cpp ucontext/ucontext-portable-test.cpp - containers/intrusive-list-test.cpp) + containers/intrusive-list-test.cpp + containers/object-pool-test.cpp) allow_deprecated_declarations(${COMMON_TESTS_SOURCES}/algorithms/projections-test.cpp) diff --git a/common/containers/object-pool-test.cpp b/common/containers/object-pool-test.cpp new file mode 100644 index 0000000000..66b12ad181 --- /dev/null +++ b/common/containers/object-pool-test.cpp @@ -0,0 +1,293 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include +#include + +#include + +#include "common/containers/object-pool.h" + +namespace { + +template +struct counting_allocator : std::allocator { + using std::allocator::allocator; + + template + struct rebind { + using other = counting_allocator; + }; + + static inline size_t allocate_calls{0}; + static inline size_t deallocate_calls{0}; + + auto allocate(std::size_t n) -> T* { + ++allocate_calls; + return std::allocator::allocate(n); + } + + auto deallocate(T* p, std::size_t n) -> void { + ++deallocate_calls; + std::allocator::deallocate(p, n); + } +}; + +template +struct tagged_allocator : std::allocator { + using std::allocator::allocator; + + template + struct rebind { + using other = tagged_allocator; + }; + + static inline std::vector allocation_tags; + + int m_tag{0}; + + tagged_allocator() = default; + explicit tagged_allocator(int tag) noexcept + : m_tag{tag} {} + + template + tagged_allocator(const tagged_allocator& other) noexcept + : m_tag{other.m_tag} {} + + auto allocate(std::size_t n) -> T* { + allocation_tags.push_back(m_tag); + return std::allocator::allocate(n); + } +}; + +struct point { + int x{}; + int y{}; + + point() = default; + point(int x_val, int y_val) noexcept + : x{x_val}, + y{y_val} {} +}; + +struct dtor_tracker { + bool* m_destroyed{nullptr}; + + explicit dtor_tracker(bool* destroyed) noexcept + : m_destroyed{destroyed} {} + + ~dtor_tracker() { + *m_destroyed = true; + } +}; + +} // namespace + +TEST(object_pool_test, acquire_forward_no_constructor_arguments) { + vk::object_pool pool{4}; + + int& x{pool.acquire()}; + + ASSERT_EQ(x, 0); + + pool.release(x); +} + +TEST(object_pool_test, acquire_forwards_single_constructor_argument) { + vk::object_pool pool{4}; + + int& x{pool.acquire(42)}; + + ASSERT_EQ(x, 42); + + pool.release(x); +} + +TEST(object_pool_test, acquire_forwards_multiple_constructor_arguments) { + vk::object_pool pool{4}; + + point& p{pool.acquire(1, 2)}; + + ASSERT_EQ(p.x, 1); + ASSERT_EQ(p.y, 2); + + pool.release(p); +} + +TEST(object_pool_test, acquire_forwards_move_only_arguments) { + vk::object_pool> pool{4}; + + std::unique_ptr& obj{pool.acquire(std::make_unique(5))}; + + ASSERT_NE(obj, nullptr); + ASSERT_EQ(*obj, 5); + + pool.release(obj); +} + +TEST(object_pool_test, release_invokes_destructor) { + vk::object_pool pool{4}; + bool destroyed{false}; + + dtor_tracker& obj{pool.acquire(&destroyed)}; + + ASSERT_FALSE(destroyed); + + pool.release(obj); + + ASSERT_TRUE(destroyed); +} + +TEST(object_pool_test, acquire_after_release_reuses_last_freed_slot) { + vk::object_pool pool{4}; + + int& a{pool.acquire(1)}; + int* addr_a{std::addressof(a)}; + pool.release(a); + + int& b{pool.acquire(2)}; + + ASSERT_EQ(std::addressof(b), addr_a); + + pool.release(b); +} + +TEST(object_pool_test, constructor_allocates_exactly_one_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{3}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + int& x{pool.acquire(7)}; + pool.release(x); +} + +TEST(object_pool_test, constructor_with_allocator_allocates_exactly_one_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{3, counting_allocator{}}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + int& x{pool.acquire(7)}; + pool.release(x); +} + +TEST(object_pool_test, constructor_with_allocator_stores_the_passed_allocator_instance) { + tagged_allocator::allocation_tags.clear(); + + vk::object_pool> pool{2, tagged_allocator{42}}; + + std::vector ptrs; + for (int i = 0; i < 5; ++i) { + ptrs.push_back(std::addressof(pool.acquire(i))); + } + + ASSERT_FALSE(tagged_allocator::allocation_tags.empty()); + for (int tag : tagged_allocator::allocation_tags) { + ASSERT_EQ(tag, 42); + } + + for (auto* p : ptrs) { + pool.release(*p); + } +} + +TEST(object_pool_test, acquiring_exactly_chunk_size_objects_does_not_grow) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{4}; + std::vector ptrs; + for (int i = 0; i < 4; ++i) { + ptrs.push_back(std::addressof(pool.acquire(i))); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + for (auto* p : ptrs) { + pool.release(*p); + } +} + +TEST(object_pool_test, release_then_acquire_does_not_allocate_a_new_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{2}; + int& a{pool.acquire(1)}; + int& b{pool.acquire(2)}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + pool.release(a); + int& c{pool.acquire(3)}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + pool.release(b); + pool.release(c); +} + +TEST(object_pool_test, allocates_chunks_when_exchausted) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{2}; + std::vector ptrs; + for (int i = 0; i < 5; ++i) { + ptrs.push_back(std::addressof(pool.acquire(i))); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 3); + + for (auto* p : ptrs) { + pool.release(*p); + } +} + +TEST(object_pool_test, destructor_deallocates_every_allocated_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + { + vk::object_pool> pool{2}; + std::vector ptrs; + for (int i = 0; i < 5; ++i) { + ptrs.push_back(std::addressof(pool.acquire(i))); + } + for (auto* p : ptrs) { + pool.release(*p); + } + } + + ASSERT_EQ(counting_allocator::allocate_calls, counting_allocator::deallocate_calls); + ASSERT_GE(counting_allocator::allocate_calls, 3); +} + +TEST(object_pool_test, acquires_returns_distinct_objects) { + constexpr int total{10}; + vk::object_pool pool{2}; + std::vector ptrs; + + for (int i = 0; i < total; ++i) { + ptrs.push_back(std::addressof(pool.acquire(i))); + } + + std::set unique_ptrs(ptrs.begin(), ptrs.end()); + ASSERT_EQ(unique_ptrs.size(), total); + + for (int i = 0; i < total; ++i) { + ASSERT_EQ(*ptrs[i], i); + } + + for (auto* p : ptrs) { + pool.release(*p); + } +} diff --git a/common/containers/object-pool.h b/common/containers/object-pool.h new file mode 100644 index 0000000000..a56982472f --- /dev/null +++ b/common/containers/object-pool.h @@ -0,0 +1,109 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +#include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" + +namespace vk { + +template> +class object_pool : private vk::not_copyable, private std::allocator_traits::template rebind_alloc { + using byte_allocator = typename std::allocator_traits::template rebind_alloc; + using byte_allocator_traits = std::allocator_traits; + + union object_pool_slot { + object_pool_slot* m_next; + T m_obj; + }; + + struct alignas(alignof(object_pool_slot)) object_pool_chunk_header { + object_pool_chunk_header* m_next{nullptr}; + }; + + std::size_t m_chunk_size{0}; + std::size_t m_chunk_byte_size{0}; + object_pool_chunk_header* m_head_chunk{nullptr}; + object_pool_slot* m_head_free_slot{nullptr}; + + auto get_byte_allocator_ref() noexcept -> byte_allocator& { + return *this; + } + + static auto slots_of(object_pool_chunk_header* chunk) noexcept -> object_pool_slot* { + return reinterpret_cast(reinterpret_cast(chunk) + sizeof(object_pool_chunk_header)); + } + + auto link_new_chunk() noexcept -> void { + std::byte* mem{byte_allocator_traits::allocate(get_byte_allocator_ref(), m_chunk_byte_size)}; + m_head_chunk = new (mem) object_pool_chunk_header{m_head_chunk}; + + object_pool_slot* slots{slots_of(m_head_chunk)}; + slots[0].m_next = nullptr; + for (size_t i = 1; i < m_chunk_size; ++i) { + slots[i].m_next = std::addressof(slots[i - 1]); + } + + m_head_free_slot = std::addressof(slots[m_chunk_size - 1]); + } + +public: + explicit object_pool(std::size_t chunk_size) noexcept + : m_chunk_size{chunk_size}, + m_chunk_byte_size{sizeof(object_pool_chunk_header) + chunk_size * sizeof(object_pool_slot)} { + assert(chunk_size > 0); + + link_new_chunk(); + } + + explicit object_pool(std::size_t chunk_size, const Allocator& allocator) noexcept + : byte_allocator(allocator), + m_chunk_size{chunk_size}, + m_chunk_byte_size{sizeof(object_pool_chunk_header) + chunk_size * sizeof(object_pool_slot)} { + assert(chunk_size > 0); + + link_new_chunk(); + } + + // All acquired objects must be released before destructor call + ~object_pool() { + auto* curr_chunk{m_head_chunk}; + while (curr_chunk != nullptr) { + auto* next_chunk{curr_chunk->m_next}; + byte_allocator_traits::deallocate(get_byte_allocator_ref(), reinterpret_cast(curr_chunk), m_chunk_byte_size); + curr_chunk = next_chunk; + } + } + + // All acquired objects must be released + template + auto acquire(Args&&... args) noexcept -> T& { + if (unlikely(m_head_free_slot == nullptr)) { + link_new_chunk(); + } + + object_pool_slot* free_slot{m_head_free_slot}; + m_head_free_slot = m_head_free_slot->m_next; + + byte_allocator_traits::construct(get_byte_allocator_ref(), std::addressof(free_slot->m_obj), std::forward(args)...); + + return free_slot->m_obj; + } + + auto release(T& obj) noexcept -> void { + byte_allocator_traits::destroy(get_byte_allocator_ref(), std::addressof(obj)); + + auto* slot{reinterpret_cast(std::addressof(obj))}; + slot->m_next = m_head_free_slot; + m_head_free_slot = slot; + } +}; + +} // namespace vk diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index ee9bd862a3..4e268fd83a 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -932,7 +932,7 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ if (mode == func_call_mode::fork_call) { if (func->is_interruptible) { - W << "(kphp::forks::start(" << FunctionName(func); + W << "(kphp::forks::start(FORK_TASK(" << FunctionName(func); } else { W << FunctionForkName(func); } @@ -969,7 +969,7 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ W << ")"; if (func->is_interruptible) { if (mode == func_call_mode::fork_call) { - W << "))"; + W << ")))"; } else { W << ")"; } @@ -2428,8 +2428,7 @@ void compile_common_op(VertexPtr root, CodeGenerator& W) { } case op_arr_acc_set_return: { auto v = root.as(); - W << "ARR_ACC_SET_RETURN" << MacroBegin{} << v->obj() << ", " << v->offset() << ", " << v->value() << ", " - << "f$" << v->set_method->name << MacroEnd{}; + W << "ARR_ACC_SET_RETURN" << MacroBegin{} << v->obj() << ", " << v->offset() << ", " << v->value() << ", " << "f$" << v->set_method->name << MacroEnd{}; break; } case op_arr_acc_check_and_get: { @@ -2441,9 +2440,7 @@ void compile_common_op(VertexPtr root, CodeGenerator& W) { W << "ARR_ACC_GET_IF_ISSET"; } - W << MacroBegin{} << v->obj() << ", " << v->offset() << ", " - << "f$" << v->check_method->name << ", " - << "f$" << v->get_method->name << MacroEnd{}; + W << MacroBegin{} << v->obj() << ", " << v->offset() << ", " << "f$" << v->check_method->name << ", " << "f$" << v->get_method->name << MacroEnd{}; break; } default: diff --git a/runtime-common/core/memory-resource/chunk-pool-resource.h b/runtime-common/core/memory-resource/chunk-pool-resource.h new file mode 100644 index 0000000000..5970d8dad2 --- /dev/null +++ b/runtime-common/core/memory-resource/chunk-pool-resource.h @@ -0,0 +1,108 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include + +#include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" +#include "runtime-common/core/memory-resource/memory_resource.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace memory_resource { + +class chunk_pool_resource : private vk::not_copyable { + struct buffer_list_node { + buffer_list_node* next{nullptr}; + }; + + struct chunk_free_list_node { + chunk_free_list_node* next{nullptr}; + }; + + std::size_t m_chunk_size{0}; + chunk_free_list_node* m_head_chunk{nullptr}; + buffer_list_node* m_head_buffer{nullptr}; + + auto link_new_buffer(void* buffer, size_t buffer_size) noexcept -> void { + new (buffer) buffer_list_node{m_head_buffer}; + + std::byte* first_chunk{static_cast(buffer) + buffer_header_size()}; + + size_t payload_size{buffer_size - buffer_header_size()}; + size_t used_buffer_size{payload_size - payload_size % m_chunk_size}; + + std::byte* curr_chunk{first_chunk}; + std::byte* last_chunk{curr_chunk + used_buffer_size - m_chunk_size}; + while (curr_chunk != last_chunk) { + std::byte* next_chunk = curr_chunk + m_chunk_size; + new (curr_chunk) chunk_free_list_node{reinterpret_cast(next_chunk)}; + curr_chunk = next_chunk; + } + + new (last_chunk) chunk_free_list_node{m_head_chunk}; + + m_head_chunk = reinterpret_cast(first_chunk); + } + +public: + auto init(void* buffer, size_t buffer_size, size_t chunk_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= chunk_size + buffer_header_size() && + reinterpret_cast(buffer) % alignof(buffer_list_node) == 0 && chunk_size >= chunk_header_size() && + chunk_size % alignof(chunk_free_list_node) == 0); + + m_chunk_size = chunk_size; + link_new_buffer(buffer, buffer_size); + m_head_buffer = static_cast(buffer); + } + + auto allocate() noexcept -> void* { + if (unlikely(m_head_chunk == nullptr)) { + return nullptr; + } + + void* allocated_chunk{m_head_chunk}; + m_head_chunk = m_head_chunk->next; + + return allocated_chunk; + } + + auto allocate0() noexcept -> void* { + void* allocated_chunk{allocate()}; + if (likely(allocated_chunk != nullptr)) { + memset(allocated_chunk, 0x00, m_chunk_size); + } + + return allocated_chunk; + } + + auto deallocate(void* mem) noexcept -> void { + m_head_chunk = new (mem) chunk_free_list_node{m_head_chunk}; + } + + auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= m_chunk_size + buffer_header_size() && + reinterpret_cast(buffer) % alignof(buffer_list_node) == 0); + + link_new_buffer(buffer, buffer_size); + m_head_buffer = static_cast(buffer); + } + + auto get_buffer_list_head() const noexcept -> buffer_list_node* { + return m_head_buffer; + } + + static auto chunk_header_size() noexcept -> std::size_t { + return sizeof(chunk_free_list_node); + } + + static auto buffer_header_size() noexcept -> std::size_t { + return sizeof(buffer_list_node); + } +}; + +} // namespace memory_resource diff --git a/runtime-common/core/memory-resource/segmented-stack-resource.h b/runtime-common/core/memory-resource/segmented-stack-resource.h new file mode 100644 index 0000000000..9817c2c6e9 --- /dev/null +++ b/runtime-common/core/memory-resource/segmented-stack-resource.h @@ -0,0 +1,117 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace memory_resource { + +template +class segmented_stack_resource : private vk::not_copyable { + struct segment_list_node { + std::byte* m_curr{nullptr}; + segment_list_node* m_next{nullptr}; + }; + + std::size_t m_segment_size{0}; + SegmentPool m_segment_pool; + segment_list_node* m_head{nullptr}; + std::byte* m_segment_begin{nullptr}; + std::byte* m_segment_curr{nullptr}; + + auto switch_to_new_segment(void* segment) noexcept -> void { + if (m_head != nullptr) { + m_head->m_curr = m_segment_curr; + } + + m_segment_begin = static_cast(segment) + segment_header_size(); + m_segment_curr = m_segment_begin; + + new (segment) segment_list_node{m_segment_begin, m_head}; + m_head = static_cast(segment); + } + + auto switch_to_old_segment() noexcept -> void { + segment_list_node* top_segment{m_head}; + m_head = m_head->m_next; + m_segment_pool.deallocate(top_segment); + + if (m_head != nullptr) { + m_segment_begin = reinterpret_cast(m_head) + segment_header_size(); + m_segment_curr = m_head->m_curr; + } else { + m_segment_begin = nullptr; + m_segment_curr = nullptr; + } + } + +public: + auto init(void* buffer, size_t buffer_size, size_t segment_size) noexcept -> void { + php_assert(segment_size > 0); + + m_segment_size = segment_size; + m_segment_pool.init(buffer, buffer_size, m_segment_size + segment_header_size()); + } + + auto allocate(size_t size) noexcept -> void* { + if (unlikely(size > m_segment_size)) { + return nullptr; + } + + if (m_head == nullptr || size > m_segment_size - static_cast(m_segment_curr - m_segment_begin)) { + void* new_segment{m_segment_pool.allocate()}; + if (unlikely(new_segment == nullptr)) { + return nullptr; + } + + php_assert(reinterpret_cast(new_segment) % alignof(segment_list_node) == 0); + + switch_to_new_segment(new_segment); + } + + void* allocated{m_segment_curr}; + m_segment_curr += size; + + return allocated; + } + + auto allocate0(size_t size) noexcept -> void* { + void* allocated{allocate(size)}; + if (likely(allocated != nullptr)) { + memset(allocated, 0x00, size); + } + + return allocated; + } + + auto deallocate(void* mem, size_t size) noexcept -> void { + m_segment_curr -= size; + + php_assert(static_cast(mem) == m_segment_curr); + + if (m_segment_curr == m_segment_begin) { + switch_to_old_segment(); + } + } + + auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { + m_segment_pool.add_extra_memory(buffer, buffer_size); + } + + auto get_buffer_list_head() noexcept -> auto* { + return m_segment_pool.get_buffer_list_head(); + } + + static auto segment_header_size() noexcept -> size_t { + return sizeof(segment_list_node); + } +}; + +} // namespace memory_resource diff --git a/runtime-light/components/confdata/confdata.cmake b/runtime-light/components/confdata/confdata.cmake index 2b955c879c..cb579a7215 100644 --- a/runtime-light/components/confdata/confdata.cmake +++ b/runtime-light/components/confdata/confdata.cmake @@ -9,6 +9,9 @@ set(KPHP_CONFDATA_COMPONENT_SRC ${RUNTIME_LIGHT_DIR}/stdlib/confdata/confdata-keys.cpp ${RUNTIME_LIGHT_DIR}/stdlib/confdata/predefined-wildcards.cpp) +set(KPHP_CONFDATA_COROUTINE_SRC ${RUNTIME_LIGHT_COROUTINE_SRC}) +list(TRANSFORM KPHP_CONFDATA_COROUTINE_SRC PREPEND "${RUNTIME_LIGHT_DIR}/") + set(KPHP_CONFDATA_TL_SRC ${RUNTIME_LIGHT_DIR}/tl/tl-types.cpp ${RUNTIME_LIGHT_DIR}/tl/tl-functions.cpp) @@ -32,6 +35,7 @@ list(TRANSFORM KPHP_CONFDATA_RUNTIME_CORE_SRC PREPEND "${RUNTIME_COMMON_DIR}/") set(KPHP_CONFDATA_SRC ${KPHP_CONFDATA_COMPONENT_SRC} + ${KPHP_CONFDATA_COROUTINE_SRC} ${KPHP_CONFDATA_TL_SRC} ${KPHP_CONFDATA_ALLOCATOR_SRC} ${KPHP_CONFDATA_DIAGNOSTICS_SRC} diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index 69355c8c7d..db9f74aabc 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -24,15 +24,21 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view PREDEFINED_WILDCARDS_ARG{"predefined-wildcards"}; static constexpr std::string_view INITIAL_INSTANCE_MEMORY_SIZE_ARG{"initial-instance-memory-size"}; static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG{"min-instance-extra-memory-size"}; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE{static_cast(1024U * 1024U)}; // 1MiB - static constexpr auto DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE{static_cast(64U * 1024U * 1024U)}; // 64MiB - static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_SIZE{64U * 1024U * 1024U}; // 64MiB + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE{static_cast(1024U * 1024U)}; // 1MiB + static constexpr auto MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE{static_cast(64U * 1024U * 1024U)}; // 64MiB + static constexpr auto DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE{static_cast(16U * 1024U * 1024U)}; // 16MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE{static_cast(1U * 1024U * 1024U)}; // 1MiB + static constexpr auto DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE = static_cast(32U * 1024U); // 32KiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE = static_cast(80U); + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB /** Leaves the same five-percent runway between legacy kPHP's hard OOM threshold and its memory limit. */ static constexpr size_t DEFAULT_OOM_HANDLING_SIZE_DIVISOR{20}; public: - AllocatorState m_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState m_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; private: const uint32_t m_argc{k2::args_count()}; @@ -47,7 +53,13 @@ struct ComponentState final : private vk::not_copyable { kphp::stl::string m_confdata_proxy_actor_name; kphp::stl::vector m_predefined_wildcards; size_t m_initial_instance_memory_size{DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE}; - size_t m_min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_SIZE}; + size_t m_min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE}; + size_t m_initial_instance_coroutine_memory_size{DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + size_t m_min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; + size_t m_initial_instance_task_memory_size{DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE}; + size_t m_instance_task_allocator_segment_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE}; + size_t m_instance_task_allocator_stack_pool_chunk_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE}; + size_t m_min_instance_extra_task_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE}; // === METHODS ================================================================================== ComponentState() noexcept; diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 85c09b9248..56f738a197 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -39,10 +39,6 @@ struct InstanceState final : vk::not_copyable { class reader_session; - // === MEMBERS ================================================================================== - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE{static_cast(16U * 1024U * 1024U)}; // 16MiB - static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE{static_cast(1U * 1024U * 1024U)}; // 1MiB - const ComponentState& m_component_state{ComponentState::get()}; public: @@ -65,7 +61,14 @@ struct InstanceState final : vk::not_copyable { std::optional m_update_failure_old_offset; // We only report the last old offset public: - kphp::coro::instance_state m_coroutine_instance_state{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE, DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE, 0}; + kphp::coro::instance_state m_coroutine_instance_state{m_component_state.m_initial_instance_coroutine_memory_size, + m_component_state.m_min_instance_extra_coroutine_memory_size, + 0, + m_component_state.m_initial_instance_task_memory_size, + m_component_state.m_instance_task_allocator_segment_size, + m_component_state.m_instance_task_allocator_stack_pool_chunk_size, + m_component_state.m_min_instance_extra_task_memory_size, + 0}; kphp::coro::io_scheduler m_io_scheduler{m_coroutine_instance_state}; StringInstanceState m_string_state; diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index 251f73d35d..6c78c5fc29 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -150,6 +150,42 @@ void ComponentState::parse_min_instance_extra_coroutine_memory_size_arg(std::str kphp::log::info("set min instance extra coroutine memory size to {} bytes", min_instance_extra_coroutine_memory_size); } +void ComponentState::parse_initial_instance_task_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse initial instance task memory size, got {}", value_view); + } + initial_instance_task_memory_size = *parsed; + kphp::log::info("set initial instance task memory size to {} bytes", initial_instance_task_memory_size); +} + +void ComponentState::parse_instance_task_allocator_segment_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse instance task allocator segment size, got {}", value_view); + } + instance_task_allocator_segment_size = *parsed; + kphp::log::info("set instance task allocator segment size to {} bytes", instance_task_allocator_segment_size); +} + +void ComponentState::parse_instance_task_allocator_stack_pool_chunk_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse instance task allocator stack pool chunk size, got {}", value_view); + } + instance_task_allocator_stack_pool_chunk_size = *parsed; + kphp::log::info("set instance task allocator stack pool chunk size to {} objects", instance_task_allocator_stack_pool_chunk_size); +} + +void ComponentState::parse_min_instance_extra_task_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse min instance extra task memory size, got {}", value_view); + } + min_instance_extra_task_memory_size = *parsed; + kphp::log::info("set min instance extra task memory size to {} bytes", min_instance_extra_task_memory_size); +} + void ComponentState::parse_args() noexcept { for (auto i = 0; i < argc; ++i) { const auto [arg_key, arg_value]{k2::arg_fetch(i)}; @@ -174,6 +210,14 @@ void ComponentState::parse_args() noexcept { parse_initial_instance_coroutine_memory_size_arg(value_view); } else if (key_view == MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG) { parse_min_instance_extra_coroutine_memory_size_arg(value_view); + } else if (key_view == INITIAL_INSTANCE_TASK_MEMORY_SIZE_ARG) { + parse_initial_instance_task_memory_size_arg(value_view); + } else if (key_view == INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE_ARG) { + parse_instance_task_allocator_segment_size_arg(value_view); + } else if (key_view == INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE_ARG) { + parse_instance_task_allocator_stack_pool_chunk_size_arg(value_view); + } else if (key_view == MIN_INSTANCE_EXTRA_TASK_MEMORY_SIZE_ARG) { + parse_min_instance_extra_task_memory_size_arg(value_view); } else { kphp::log::warning("unexpected argument format: {}", key_view); } diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index f737cd5c62..97be4f5028 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -19,7 +19,7 @@ #include "runtime-light/stdlib/kml/kml-state.h" struct ComponentState final : private vk::not_copyable { - AllocatorState component_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState component_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; KmlComponentState kml_component_state; // This member does not hold any KPHP types, so setting a reference counter is unnecessary. const uint32_t argc{k2::args_count()}; @@ -29,10 +29,14 @@ struct ComponentState final : private vk::not_copyable { mixed runtime_config; string cluster_name{DEFAULT_CLUSTER_NAME.data(), DEFAULT_CLUSTER_NAME.size()}; bool exit_after_response{}; - uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; + uint64_t initial_instance_memory_size{DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE}; - uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + uint64_t initial_instance_coroutine_memory_size{DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; + uint64_t initial_instance_task_memory_size{DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE}; + uint64_t instance_task_allocator_segment_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE}; + uint64_t instance_task_allocator_stack_pool_chunk_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE}; + uint64_t min_instance_extra_task_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE}; ComponentState() noexcept { parse_env(); @@ -67,12 +71,20 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; + static constexpr std::string_view INITIAL_INSTANCE_TASK_MEMORY_SIZE_ARG = "initial-instance-task-memory-size"; + static constexpr std::string_view INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE_ARG = "instance-task-allocator-segment-size"; + static constexpr std::string_view INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE_ARG = "instance-task-allocator-stack-pool-chunk-size"; + static constexpr std::string_view MIN_INSTANCE_EXTRA_TASK_MEMORY_SIZE_ARG = "min-instance-extra-task-memory-size"; static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB - static constexpr auto DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB + static constexpr auto DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE = static_cast(32U * 1024U); // 32KiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE = static_cast(80U); + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB void parse_env() noexcept; @@ -95,4 +107,12 @@ struct ComponentState final : private vk::not_copyable { void parse_initial_instance_coroutine_memory_size_arg(std::string_view) noexcept; void parse_min_instance_extra_coroutine_memory_size_arg(std::string_view) noexcept; + + void parse_initial_instance_task_memory_size_arg(std::string_view) noexcept; + + void parse_instance_task_allocator_segment_size_arg(std::string_view) noexcept; + + void parse_instance_task_allocator_stack_pool_chunk_size_arg(std::string_view) noexcept; + + void parse_min_instance_extra_task_memory_size_arg(std::string_view) noexcept; }; diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 960cf6b113..ce295fc7d6 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -93,7 +93,13 @@ struct InstanceState final : vk::not_copyable { kphp::log::contextual_tags instance_tags; kphp::coro::instance_state coroutine_instance_state{component_state.initial_instance_coroutine_memory_size, - component_state.min_instance_extra_coroutine_memory_size, 0}; + component_state.min_instance_extra_coroutine_memory_size, + 0, + component_state.initial_instance_task_memory_size, + component_state.instance_task_allocator_segment_size, + component_state.instance_task_allocator_stack_pool_chunk_size, + component_state.min_instance_extra_task_memory_size, + 0}; kphp::coro::io_scheduler io_scheduler{coroutine_instance_state}; ForkInstanceState fork_instance_state; WaitQueueInstanceState wait_queue_instance_state; diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 887fb7b2a4..1749bf42ba 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -8,23 +8,23 @@ #include "common/mixin/not_copyable.h" -#include "runtime-common/core/allocator/pool-allocator.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" namespace kphp::coro { struct instance_state final : private vk::not_copyable { - - instance_state(size_t coroutine_mem_size, size_t min_extra_coroutine_mem_size, size_t oom_handling_coroutine_mem_size) noexcept - : coroutine_allocator{coroutine_mem_size, min_extra_coroutine_mem_size, oom_handling_coroutine_mem_size} {} + instance_state(size_t coroutine_mem_size, size_t min_extra_coroutine_mem_size, size_t oom_handling_coroutine_mem_size, size_t task_mem_size, + size_t task_allocator_segment_size, size_t task_allocator_stack_pool_chunk_size, size_t min_extra_task_mem_size, + size_t oom_handling_task_mem_size) noexcept + : coroutine_allocator{coroutine_mem_size, min_extra_coroutine_mem_size, oom_handling_coroutine_mem_size}, + task_allocator{task_mem_size, task_allocator_segment_size, task_allocator_stack_pool_chunk_size, min_extra_task_mem_size, oom_handling_task_mem_size} {} static instance_state& get() noexcept; - static kphp::memory::pool_allocator& get_coroutine_allocator() noexcept { - return kphp::coro::instance_state::get().coroutine_allocator; - } - - kphp::memory::pool_allocator coroutine_allocator; + kphp::coro::detail::memory::runtime_coroutine_allocator coroutine_allocator; + kphp::coro::detail::memory::task_allocator task_allocator; kphp::coro::async_stack_root coroutine_stack_root; }; diff --git a/runtime-light/coroutine/coroutine.cmake b/runtime-light/coroutine/coroutine.cmake new file mode 100644 index 0000000000..4d6cfd8a58 --- /dev/null +++ b/runtime-light/coroutine/coroutine.cmake @@ -0,0 +1,3 @@ +set(RUNTIME_LIGHT_COROUTINE_SRC + coroutine/detail/allocator/task-allocator.cpp + coroutine/detail/allocator/runtime-coroutine-allocator.cpp) \ No newline at end of file diff --git a/runtime-light/coroutine/detail/allocator/coroutine-allocator.h b/runtime-light/coroutine/detail/allocator/coroutine-allocator.h index 48c495d81f..8246e83e6f 100644 --- a/runtime-light/coroutine/detail/allocator/coroutine-allocator.h +++ b/runtime-light/coroutine/detail/allocator/coroutine-allocator.h @@ -6,7 +6,7 @@ #include -#include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" namespace kphp::coro::detail::memory { @@ -20,11 +20,11 @@ struct coroutine_allocator { explicit coroutine_allocator(const coroutine_allocator& /*unused*/) noexcept {} constexpr value_type* allocate(size_t n) noexcept { - return static_cast(kphp::coro::instance_state::get_coroutine_allocator().alloc_script_memory(n * sizeof(T))); + return static_cast(kphp::coro::detail::memory::runtime_coroutine_allocator::get().alloc_script_memory(n * sizeof(T))); } constexpr void deallocate(T* p, size_t n) noexcept { - kphp::coro::instance_state::get_coroutine_allocator().free_script_memory(p, n * sizeof(T)); + kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(p, n * sizeof(T)); } }; diff --git a/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h b/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h index 6ad337c955..fc25dd7099 100644 --- a/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h +++ b/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h @@ -8,32 +8,32 @@ #include #include "runtime-common/core/allocator/details/malloc-interface.h" -#include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" namespace kphp::coro::detail::memory { inline auto alloc(size_t size) noexcept -> void* { - return kphp::memory::details::malloc_interface::alloc(size); + return kphp::memory::details::malloc_interface::alloc(size); } inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { - return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); } inline auto calloc(size_t num, size_t size) noexcept -> void* { - return kphp::memory::details::malloc_interface::calloc(num, size); + return kphp::memory::details::malloc_interface::calloc(num, size); } inline auto free(void* ptr) noexcept -> void { - kphp::memory::details::malloc_interface::free(ptr); + kphp::memory::details::malloc_interface::free(ptr); } inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { - return kphp::memory::details::malloc_interface::realloc(ptr, new_size); + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); } inline auto strdup(const char* str1) noexcept -> char* { - return kphp::memory::details::malloc_interface::strdup(str1); + return kphp::memory::details::malloc_interface::strdup(str1); } } // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp new file mode 100644 index 0000000000..4937eee417 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp @@ -0,0 +1,15 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" + +#include "runtime-light/coroutine/coroutine-state.h" + +namespace kphp::coro::detail::memory { + +auto runtime_coroutine_allocator::get() noexcept -> runtime_coroutine_allocator& { + return kphp::coro::instance_state::get().coroutine_allocator; +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h new file mode 100644 index 0000000000..99f0cc0fd4 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h @@ -0,0 +1,44 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-common/core/allocator/pool-allocator.h" + +namespace kphp::coro::detail::memory { + +struct runtime_coroutine_allocator final { +private: + kphp::memory::pool_allocator m_allocator; + +public: + static auto get() noexcept -> runtime_coroutine_allocator&; + + runtime_coroutine_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} + + auto free() noexcept -> void { + m_allocator.free(); + } + + auto alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc_script_memory(size); + } + + auto calloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.calloc_script_memory(size); + } + + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc_script_memory(mem, new_size, old_size); + } + + auto free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free_script_memory(mem, size); + } +}; + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.cpp b/runtime-light/coroutine/detail/allocator/task-allocator.cpp new file mode 100644 index 0000000000..82d6a2e3b5 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/task-allocator.cpp @@ -0,0 +1,15 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" + +#include "runtime-light/coroutine/coroutine-state.h" + +namespace kphp::coro::detail::memory { + +auto task_allocator::get() noexcept -> task_allocator& { + return kphp::coro::instance_state::get().task_allocator; +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h new file mode 100644 index 0000000000..63d936613e --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -0,0 +1,182 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/containers/object-pool.h" +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" +#include "runtime-common/core/memory-resource/chunk-pool-resource.h" +#include "runtime-common/core/memory-resource/segmented-stack-resource.h" +#include "runtime-light/coroutine/detail/allocator/coroutine-allocator.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::coro::detail::memory { + +struct task_allocator final : private vk::not_copyable { + struct shared_chunk_pool { + memory_resource::chunk_pool_resource& m_chunk_pool; + + shared_chunk_pool() noexcept + : m_chunk_pool{task_allocator::get().m_chunk_pool} {} + + auto init(void* /*unused*/, size_t /*unused*/, size_t /*unused*/) noexcept -> void {} + + auto allocate() noexcept -> void* { + return m_chunk_pool.allocate(); + } + + auto allocate0() noexcept -> void* { + return m_chunk_pool.allocate0(); + } + + auto deallocate(void* mem) noexcept -> void { + m_chunk_pool.deallocate(mem); + } + + auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { + m_chunk_pool.add_extra_memory(buffer, buffer_size); + } + + auto get_buffer_list_head() const noexcept -> auto* { + return m_chunk_pool.get_buffer_list_head(); + } + }; + +private: + vk::object_pool, + kphp::coro::detail::memory::coroutine_allocator>> + m_stack_pool; + memory_resource::chunk_pool_resource m_chunk_pool; + memory_resource::segmented_stack_resource* m_curr_stack{nullptr}; + size_t m_segment_size{0}; + size_t m_min_extra_mem_size{0}; + + auto request_extra_memory(size_t requested_size) noexcept -> void { + size_t extra_mem_size{std::max(m_min_extra_mem_size, requested_size)}; + // Take into account internal layout of header for buffer + extra_mem_size += sizeof(memory_resource::chunk_pool_resource::buffer_header_size()); + // The smallest power of two that is not smaller than `extra_mem_size` + extra_mem_size = std::bit_ceil(extra_mem_size); + + auto* extra_mem{kphp::memory::platform::alloc(extra_mem_size)}; + + kphp::log::assertion(extra_mem != nullptr); + + m_chunk_pool.add_extra_memory(extra_mem, extra_mem_size); + } + +public: + static auto get() noexcept -> task_allocator&; + + task_allocator(size_t script_mem_size, size_t segment_size, size_t stack_pool_chunk_size, size_t min_extra_mem_size, size_t /*unused*/) noexcept + : m_stack_pool{stack_pool_chunk_size}, + m_segment_size{segment_size}, + m_min_extra_mem_size{min_extra_mem_size} { + void* buffer{kphp::memory::platform::alloc(script_mem_size)}; + + kphp::log::assertion(buffer != nullptr); + + m_chunk_pool.init(buffer, script_mem_size, m_segment_size + memory_resource::segmented_stack_resource::segment_header_size()); + } + + auto free() noexcept -> void { + auto* curr_buffer{m_chunk_pool.get_buffer_list_head()}; + while (curr_buffer != nullptr) { + auto* next_buffer = curr_buffer->next; + kphp::memory::platform::free(curr_buffer); + curr_buffer = next_buffer; + } + } + + auto acquire_stack() noexcept -> memory_resource::segmented_stack_resource& { + auto& stack{m_stack_pool.acquire()}; + // We can pass nullptr as buffer and 0 as buffer_size, because init in shared_chunk_pool is noop + stack.init(nullptr, 0, m_segment_size); + + return stack; + } + + auto release_stack(memory_resource::segmented_stack_resource& stack) noexcept -> void { + m_stack_pool.release(stack); + } + + auto segment_size() const noexcept -> size_t { + return m_segment_size; + } + + auto current_stack() const noexcept -> memory_resource::segmented_stack_resource* { + return m_curr_stack; + } + + auto set_stack(memory_resource::segmented_stack_resource* stack) noexcept -> void { + m_curr_stack = stack; + } + + auto exchange_stack(memory_resource::segmented_stack_resource* stack) noexcept + -> memory_resource::segmented_stack_resource* { + auto* prev{m_curr_stack}; + set_stack(stack); + + return prev; + } + + auto fork() noexcept { + struct fork_guard { + kphp::coro::detail::memory::task_allocator& m_task_allocator; + memory_resource::segmented_stack_resource* m_prev_stack{nullptr}; + + explicit fork_guard(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : m_task_allocator{task_allocator}, + m_prev_stack{task_allocator.exchange_stack(nullptr)} {} + + ~fork_guard() { + m_task_allocator.set_stack(m_prev_stack); + } + }; + + return fork_guard{*this}; + } + + auto alloc_script_memory(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + kphp::log::assertion(m_curr_stack != nullptr); + + void* mem{m_curr_stack->allocate(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = m_curr_stack->allocate(size); + + kphp::log::assertion(mem != nullptr); + } + + return mem; + } + + auto calloc_script_memory(size_t size) noexcept -> void* { + kphp::log::assertion(size != 0); + kphp::log::assertion(m_curr_stack != nullptr); + + void* mem{m_curr_stack->allocate0(size)}; + if (mem == nullptr) [[unlikely]] { + request_extra_memory(size); + mem = m_curr_stack->allocate0(size); + + kphp::log::assertion(mem != nullptr); + } + + return mem; + } + + auto free_script_memory(void* mem, size_t size) noexcept -> void { + kphp::log::assertion(size != 0); + kphp::log::assertion(m_curr_stack != nullptr); + + m_curr_stack->deallocate(mem, size); + } +}; + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h new file mode 100644 index 0000000000..7c50c719b0 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h @@ -0,0 +1,122 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::coro::detail::memory::task { + +struct control_block { +private: + static constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB + +public: + enum class backend_type : uint8_t { stack, stack_owner, coroutine_pool }; + + memory_resource::segmented_stack_resource* stack{nullptr}; + uint16_t base_offset{}; + backend_type backend{}; + + static constexpr auto max_size() noexcept -> uint64_t { + return MALLOC_REPLACER_MAX_ALLOC; + } + + static constexpr auto max_alignment() noexcept -> uint64_t { + return std::numeric_limits::max(); + } +}; + +inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { + // Check that provided alignment is power of two + const size_t align{static_cast(al)}; + if (align == 0 || !std::has_single_bit(align) || align >= kphp::coro::detail::memory::task::control_block::max_alignment()) [[unlikely]] { + php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", + kphp::coro::detail::memory::task::control_block::max_alignment(), align); + return nullptr; + } + + // Check that memory is enough + constexpr size_t cb_size{sizeof(kphp::coro::detail::memory::task::control_block)}; + if (size > kphp::coro::detail::memory::task::control_block::max_size() - (align - 1) - cb_size) [[unlikely]] { + php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); + return nullptr; + } + + // Request mem from underlying memory manager + const size_t total_size{size + (align - 1) + cb_size}; + void* base{nullptr}; + kphp::coro::detail::memory::task::control_block::backend_type backend{}; + auto& task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + if (total_size <= task_allocator.segment_size()) [[likely]] { + if (task_allocator.current_stack() == nullptr) { + auto& stack{task_allocator.acquire_stack()}; + task_allocator.set_stack(std::addressof(stack)); + backend = kphp::coro::detail::memory::task::control_block::backend_type::stack_owner; + } else { + backend = kphp::coro::detail::memory::task::control_block::backend_type::stack; + } + + base = task_allocator.alloc_script_memory(total_size); + } else { + base = kphp::coro::detail::memory::runtime_coroutine_allocator::get().alloc_script_memory(total_size); + backend = kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool; + } + + if (base == nullptr) [[unlikely]] { + php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); + return base; + } + + const uint64_t base_u{reinterpret_cast(base)}; + // The smallest multiple of `align` greater than or equal to requested memory + const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; + const uint64_t base_offset_u{aligned_u - base_u}; + + std::construct_at(reinterpret_cast(aligned_u - cb_size), task_allocator.current_stack(), // NOLINT + static_cast(base_offset_u), backend); + + return reinterpret_cast(aligned_u); // NOLINT +} + +inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void { + if (ptr == nullptr) [[unlikely]] { + return; + } + + const size_t align{static_cast(al)}; + const size_t cb_size{sizeof(kphp::coro::detail::memory::task::control_block)}; + const size_t total_size{size + (align - 1) + cb_size}; + auto* cb{reinterpret_cast(static_cast(ptr) - cb_size)}; + auto& task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + switch (cb->backend) { + case kphp::coro::detail::memory::task::control_block::backend_type::stack: + kphp::log::assertion(task_allocator.current_stack() == cb->stack); + + task_allocator.free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); + break; + case kphp::coro::detail::memory::task::control_block::backend_type::stack_owner: + kphp::log::assertion(task_allocator.current_stack() == cb->stack); + + task_allocator.free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); + task_allocator.set_stack(nullptr); + task_allocator.release_stack(*cb->stack); + break; + case kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool: + [[unlikely]] kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(reinterpret_cast(ptr) - cb->base_offset, + total_size); + break; + } +} + +} // namespace kphp::coro::detail::memory::task diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index e6148b6217..5d0c56ca20 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -18,6 +18,7 @@ #include "common/containers/intrusive-list.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -103,7 +104,9 @@ struct promise_base : kphp::coro::async_stack_element { m_state = vk::intrusive::list>>{}; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto& async_stack_root{*get_async_stack_frame().async_stack_root}; + auto* prev_stack{m_task_allocator.exchange_stack(nullptr)}; kphp::coro::resume(handle, async_stack_root); + m_task_allocator.set_stack(prev_stack); } // coroutine already completed, don't suspend @@ -140,6 +143,7 @@ struct promise_base : kphp::coro::async_stack_element { private: uint32_t m_refcnt{1}; std::variant>>> m_state; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; }; template diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 8e874c480c..88c31b5b6a 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -12,7 +12,8 @@ #include "common/containers/final_action.h" #include "runtime-light/coroutine/async-stack.h" -#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/detail/allocator/task-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { @@ -64,21 +65,18 @@ struct promise_base : kphp::coro::async_stack_element { return std::coroutine_handle::from_promise(*const_cast(static_cast(this))).done(); } - template - auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::coro::detail::memory::alloc(n); - } - template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::coro::detail::memory::alloc_aligned(n, al); + return kphp::coro::detail::memory::task::alloc_aligned(n, al); } - auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::coro::detail::memory::free(ptr); + auto operator delete(void* ptr, size_t n, std::align_val_t al) noexcept -> void { + kphp::coro::detail::memory::task::free_aligned(ptr, n, al); } void* m_next{}; + memory_resource::segmented_stack_resource* m_stack{}; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; }; template @@ -103,6 +101,7 @@ class awaiter_base { bool m_started{}; bool m_suspended{}; std::coroutine_handle m_coro{}; + memory_resource::segmented_stack_resource* m_prev_stack{}; public: explicit awaiter_base(std::coroutine_handle coro) noexcept @@ -131,6 +130,7 @@ class awaiter_base { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> std::coroutine_handle { + m_prev_stack = m_coro.promise().m_task_allocator.exchange_stack(m_coro.promise().m_stack); push_async_stack_frame(awaiting_coroutine.promise().get_async_stack_frame(), STACK_RETURN_ADDRESS); m_coro.promise().m_next = awaiting_coroutine.address(); m_suspended = true; @@ -138,6 +138,7 @@ class awaiter_base { } auto await_resume() noexcept -> void { + m_coro.promise().m_task_allocator.set_stack(m_prev_stack); m_suspended = false; } }; @@ -176,12 +177,16 @@ struct task { ~task() { if (m_coro) { + auto& task_allocator{m_coro.promise().task_allocator}; + auto* prev_stack{task_allocator.exchange_stack(m_coro.promise().m_stack)}; m_coro.destroy(); + task_allocator.set_stack(prev_stack); } } struct promise_base : task_impl::promise_base { auto get_return_object() noexcept -> task { + this->m_stack = this->m_task_allocator.current_stack(); return task{std::coroutine_handle::from_promise(*static_cast(this))}; } @@ -247,3 +252,5 @@ struct task { }; } // namespace kphp::coro + +#define FORK_TASK(...) (kphp::coro::detail::memory::task_allocator::get().fork(), ...); diff --git a/runtime-light/coroutine/type-traits.h b/runtime-light/coroutine/type-traits.h index 8d85fd4a54..ad2bf0569f 100644 --- a/runtime-light/coroutine/type-traits.h +++ b/runtime-light/coroutine/type-traits.h @@ -42,11 +42,17 @@ class async_function_return_type { } // namespace type_traits_impl +template +inline constexpr bool is_task_v = requires { + { static_cast>(std::declval()) }; +}; + template -requires std::invocable -inline constexpr bool is_async_function_v = requires { - { static_cast>(std::declval>()) }; -} || requires { +inline constexpr bool is_task_function_v = requires { requires std::invocable && is_task_v>; }; + +template +inline constexpr bool is_async_function_v = is_task_function_v || requires { + requires std::invocable; { static_cast>(std::declval>()) }; }; diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index c16371b353..cefb53d717 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -19,6 +19,7 @@ include(${RUNTIME_LIGHT_DIR}/allocator/allocator.cmake) include(${RUNTIME_LIGHT_DIR}/components/kphp/kphp.cmake) include(${RUNTIME_LIGHT_DIR}/core/core.cmake) include(${RUNTIME_LIGHT_DIR}/server/server.cmake) +include(${RUNTIME_LIGHT_DIR}/coroutine/coroutine.cmake) include(${RUNTIME_LIGHT_DIR}/stdlib/stdlib.cmake) include(${RUNTIME_LIGHT_DIR}/tl/tl.cmake) include(${RUNTIME_LIGHT_DIR}/memory-resource-impl/memory-resource-impl.cmake) diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index fd2c4de625..ff4133f66a 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -59,6 +59,22 @@ auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task +requires(kphp::coro::is_task_function_v) +auto id_managed(F f, Args... args) noexcept -> kphp::coro::task>::awaiter_return_type> { + auto& fork_instance_st{ForkInstanceState::get()}; + const auto saved_fork_id{fork_instance_st.current_id}; + if constexpr (std::is_void_v>::awaiter_return_type>) { + co_await std::invoke(std::move(f), std::move(args)...); + fork_instance_st.current_id = saved_fork_id; + co_return; + } else { + auto value{co_await std::invoke(std::move(f), std::move(args)...)}; + fork_instance_st.current_id = saved_fork_id; + co_return std::move(value); + } +} + template auto start(kphp::coro::task task) noexcept -> int64_t { auto& fork_instance_st{ForkInstanceState::get()}; diff --git a/tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp b/tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp new file mode 100644 index 0000000000..010a64cf38 --- /dev/null +++ b/tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp @@ -0,0 +1,205 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include + +#include + +#include "runtime-common/core/memory-resource/chunk-pool-resource.h" + +namespace { + +constexpr size_t CHUNK_SIZE{64}; +// mirrors chunk_pool_resource's internal list_node layout: a single pointer, no padding +constexpr size_t HEADER_SIZE{sizeof(void*)}; + +template +using aligned_buffer = std::array; + +} // namespace + +TEST(chunk_pool_resource_test, uninited_state) { + memory_resource::chunk_pool_resource resource; + + ASSERT_EQ(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate0(), nullptr); + ASSERT_EQ(resource.get_buffer_list_head(), nullptr); +} + +TEST(chunk_pool_resource_test, single_chunk_exact_buffer) { + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + void* mem{resource.allocate()}; + + ASSERT_EQ(mem, static_cast(buffer.data() + HEADER_SIZE)); + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, multiple_chunks_exact_fit) { + constexpr size_t CHUNKS{5}; + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + for (size_t i = 0; i < CHUNKS; ++i) { + void* mem{resource.allocate()}; + ASSERT_EQ(mem, static_cast(buffer.data() + HEADER_SIZE + i * CHUNK_SIZE)); + } + + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, truncated_tail_is_unused) { + constexpr size_t CHUNKS{3}; + constexpr size_t TAIL{CHUNK_SIZE - 1}; // smaller than one chunk, must be dropped + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + for (size_t i = 0; i < CHUNKS; ++i) { + ASSERT_NE(resource.allocate(), nullptr); + } + + // the trailing tail bytes don't make up a whole chunk, so they must not be handed out + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, minimal_chunk_size) { + constexpr size_t MIN_CHUNK_SIZE{sizeof(void*)}; + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), MIN_CHUNK_SIZE); + + void* a{resource.allocate()}; + void* b{resource.allocate()}; + + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + ASSERT_NE(a, b); + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, allocate0_returns_zeroed_memory) { + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + void* mem{resource.allocate()}; + + ASSERT_NE(mem, nullptr); + + std::memset(mem, 0xFF, CHUNK_SIZE); + resource.deallocate(mem); + void* mem0{resource.allocate0()}; + + ASSERT_EQ(mem0, mem); + + auto* bytes{static_cast(mem0)}; + for (size_t i = 0; i < CHUNK_SIZE; ++i) { + ASSERT_EQ(static_cast(bytes[i]), 0); + } +} + +TEST(chunk_pool_resource_test, allocate0_on_exhausted_pool_returns_null) { + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate0(), nullptr); +} + +TEST(chunk_pool_resource_test, chunks_do_not_overlap) { + constexpr size_t CHUNKS{4}; + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + std::array chunks{}; + for (size_t i = 0; i < CHUNKS; ++i) { + chunks[i] = resource.allocate(); + + ASSERT_NE(chunks[i], nullptr); + + std::memset(chunks[i], static_cast(i + 1), CHUNK_SIZE); + } + + for (size_t i = 0; i < CHUNKS; ++i) { + auto* bytes{static_cast(chunks[i])}; + for (size_t j = 0; j < CHUNK_SIZE; ++j) { + ASSERT_EQ(bytes[j], static_cast(i + 1)); + } + } +} + +TEST(chunk_pool_resource_test, add_extra_memory_grows_pool_after_exhaustion) { + alignas(alignof(void*)) aligned_buffer buffer1{}; + alignas(alignof(void*)) aligned_buffer buffer2{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer1.data(), buffer1.size(), CHUNK_SIZE); + + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate(), nullptr); // exhausted + + resource.add_extra_memory(buffer2.data(), buffer2.size()); + + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate(), nullptr); // exhausted again +} + +TEST(chunk_pool_resource_test, add_extra_memory_preserves_existing_free_chunks) { + constexpr size_t OLD_CHUNKS{2}; + alignas(alignof(void*)) aligned_buffer old_buffer{}; + alignas(alignof(void*)) aligned_buffer extra_buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(old_buffer.data(), old_buffer.size(), CHUNK_SIZE); + + // consume the first chunk of old_buffer, the second one is left in the free list + ASSERT_NE(resource.allocate(), nullptr); + + void* remaining_old_chunk{static_cast(old_buffer.data() + HEADER_SIZE + CHUNK_SIZE)}; + resource.add_extra_memory(extra_buffer.data(), extra_buffer.size()); + + // the newly added buffer's chunk is handed out before the old buffer's leftover chunk + ASSERT_EQ(resource.allocate(), static_cast(extra_buffer.data() + HEADER_SIZE)); + ASSERT_EQ(resource.allocate(), remaining_old_chunk); + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, buffer_list_head_tracks_most_recently_added_buffer) { + alignas(alignof(void*)) aligned_buffer buffer1{}; + alignas(alignof(void*)) aligned_buffer buffer2{}; + alignas(alignof(void*)) aligned_buffer buffer3{}; + + memory_resource::chunk_pool_resource resource; + + resource.init(buffer1.data(), buffer1.size(), CHUNK_SIZE); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer1.data())); + + resource.add_extra_memory(buffer2.data(), buffer2.size()); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer2.data())); + + resource.add_extra_memory(buffer3.data(), buffer3.size()); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer3.data())); + + auto* head{resource.get_buffer_list_head()}; + + ASSERT_EQ(static_cast(head->next), static_cast(buffer2.data())); + + head = head->next; + + ASSERT_EQ(static_cast(head->next), static_cast(buffer1.data())); + + head = head->next; + + ASSERT_EQ(static_cast(head->next), nullptr); +} diff --git a/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp b/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp new file mode 100644 index 0000000000..99abee47a3 --- /dev/null +++ b/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp @@ -0,0 +1,289 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include +#include + +#include + +#include "runtime-common/core/memory-resource/chunk-pool-resource.h" +#include "runtime-common/core/memory-resource/segmented-stack-resource.h" + +namespace { + +constexpr size_t SEGMENT_SIZE{64}; +// mirrors segment_list_node's layout: two pointers, no padding +constexpr size_t SEGMENT_HEADER_SIZE{2 * sizeof(void*)}; +// mirrors chunk_pool_resource's internal buffer_list_node layout: a single pointer, no padding +constexpr size_t POOL_HEADER_SIZE{sizeof(void*)}; +// size of one chunk as seen by the underlying SegmentPool: segment payload + segment_list_node header +constexpr size_t SEGMENT_CHUNK_SIZE{SEGMENT_SIZE + SEGMENT_HEADER_SIZE}; + +// heap-allocated to avoid -Warray-bounds false positives in gcc +[[gnu::noinline]] auto make_buffer(size_t segments) -> std::vector { + return std::vector(POOL_HEADER_SIZE + segments * SEGMENT_CHUNK_SIZE); +} + +auto segment_payload(std::byte* buffer_start, size_t index) noexcept -> std::byte* { + return buffer_start + POOL_HEADER_SIZE + index * SEGMENT_CHUNK_SIZE + SEGMENT_HEADER_SIZE; +} + +} // namespace + +TEST(segmented_stack_resource_test, uninited_state) { + memory_resource::segmented_stack_resource resource; + + ASSERT_EQ(resource.allocate(1), nullptr); + ASSERT_EQ(resource.allocate0(1), nullptr); + ASSERT_EQ(resource.get_buffer_list_head(), nullptr); +} + +TEST(segmented_stack_resource_test, single_allocation_exact_segment_fit) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* mem{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_EQ(mem, static_cast(segment_payload(buffer.data(), 0))); + // the single segment is fully consumed, and there's no more pool memory for a new one + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(segmented_stack_resource_test, sequential_allocations_bump_within_segment) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* a{resource.allocate(16)}; + void* b{resource.allocate(16)}; + void* c{resource.allocate(32)}; + + ASSERT_EQ(a, static_cast(segment_payload(buffer.data(), 0))); + ASSERT_EQ(b, static_cast(segment_payload(buffer.data(), 0) + 16)); + ASSERT_EQ(c, static_cast(segment_payload(buffer.data(), 0) + 32)); + // exactly filled the segment + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(segmented_stack_resource_test, allocation_crossing_segment_boundary_switches_segment) { + auto buffer{make_buffer(2)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* first{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_EQ(first, static_cast(segment_payload(buffer.data(), 0))); + + // doesn't fit in the remaining space of the first segment, must switch to a new one + void* second{resource.allocate(1)}; + + ASSERT_EQ(second, static_cast(segment_payload(buffer.data(), 1))); + + ASSERT_EQ(resource.allocate(SEGMENT_SIZE), nullptr); // pool exhausted +} + +TEST(segmented_stack_resource_test, lifo_deallocate_reuses_address_within_segment) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + resource.allocate(16); + void* b1{resource.allocate(16)}; + + resource.deallocate(b1, 16); + void* b2{resource.allocate(16)}; + + ASSERT_EQ(b2, b1); +} + +TEST(segmented_stack_resource_test, full_drain_returns_segment_to_pool_and_resets_state) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* mem1{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_NE(mem1, nullptr); + + resource.deallocate(mem1, SEGMENT_SIZE); + + // the segment was released back to the pool, so a fresh allocation gets the same address + void* mem2{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_EQ(mem2, mem1); +} + +TEST(segmented_stack_resource_test, regrowth_after_full_drain_across_multiple_segments) { + auto buffer{make_buffer(2)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* first{resource.allocate(SEGMENT_SIZE)}; + void* second{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + + // unwind in LIFO order + resource.deallocate(second, SEGMENT_SIZE); + resource.deallocate(first, SEGMENT_SIZE); + + void* first_again{resource.allocate(SEGMENT_SIZE)}; + void* second_again{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_EQ(first_again, first); + ASSERT_EQ(second_again, second); +} + +TEST(segmented_stack_resource_test, multi_segment_unwind_restores_saved_cursor) { + auto buffer{make_buffer(2)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* a{resource.allocate(48)}; // segment 0, [0, 48) + void* b{resource.allocate(32)}; // doesn't fit remaining 16 bytes of segment 0, switches to segment 1 + void* c{resource.allocate(16)}; // segment 1, right after b + + ASSERT_EQ(a, static_cast(segment_payload(buffer.data(), 0))); + ASSERT_EQ(b, static_cast(segment_payload(buffer.data(), 1))); + ASSERT_EQ(c, static_cast(segment_payload(buffer.data(), 1) + 32)); + + resource.deallocate(c, 16); + resource.deallocate(b, 32); // drains segment 1 back to the pool + + // segment 0's cursor (saved when we switched away from it) must be restored correctly + void* d{resource.allocate(16)}; + ASSERT_EQ(d, static_cast(segment_payload(buffer.data(), 0) + 48)); +} + +TEST(segmented_stack_resource_test, exhaustion_returns_null_without_corrupting_state) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* mem{resource.allocate(SEGMENT_SIZE - 16)}; + + ASSERT_NE(mem, nullptr); + + // doesn't fit in the remaining 16 bytes, and there's no more pool memory for a new segment + ASSERT_EQ(resource.allocate(32), nullptr); + + // the failed attempt must not have corrupted the cursor of the current segment + void* rest{resource.allocate(16)}; + + ASSERT_EQ(rest, static_cast(segment_payload(buffer.data(), 0) + (SEGMENT_SIZE - 16))); +} + +TEST(segmented_stack_resource_test, allocate0_returns_zeroed_memory) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + void* mem{resource.allocate(SEGMENT_SIZE)}; + + ASSERT_NE(mem, nullptr); + + resource.deallocate(mem, SEGMENT_SIZE); + + void* mem0{resource.allocate0(SEGMENT_SIZE)}; + + ASSERT_EQ(mem0, mem); + + auto* bytes{static_cast(mem0)}; + for (size_t i = 0; i < SEGMENT_SIZE; ++i) { + ASSERT_EQ(static_cast(bytes[i]), 0); + } +} + +TEST(segmented_stack_resource_test, allocate0_on_exhausted_pool_returns_null) { + auto buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); + ASSERT_EQ(resource.allocate0(1), nullptr); +} + +TEST(segmented_stack_resource_test, segments_do_not_overlap) { + constexpr size_t SEGMENTS{4}; + auto buffer{make_buffer(SEGMENTS)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); + + std::array segments{}; + for (size_t i = 0; i < SEGMENTS; ++i) { + segments[i] = resource.allocate(SEGMENT_SIZE); + + ASSERT_NE(segments[i], nullptr); + + std::memset(segments[i], static_cast(i + 1), SEGMENT_SIZE); + } + + for (size_t i = 0; i < SEGMENTS; ++i) { + auto* bytes{static_cast(segments[i])}; + for (size_t j = 0; j < SEGMENT_SIZE; ++j) { + ASSERT_EQ(bytes[j], static_cast(i + 1)); + } + } +} + +TEST(segmented_stack_resource_test, add_extra_memory_grows_pool_after_exhaustion) { + auto buffer1{make_buffer(1)}; + auto buffer2{make_buffer(2)}; + memory_resource::segmented_stack_resource resource; + resource.init(buffer1.data(), buffer1.size(), SEGMENT_SIZE); + + ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); + ASSERT_EQ(resource.allocate(1), nullptr); // exhausted + + resource.add_extra_memory(buffer2.data(), buffer2.size()); + + ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); + ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); + ASSERT_EQ(resource.allocate(1), nullptr); // exhausted again +} + +TEST(segmented_stack_resource_test, add_extra_memory_before_exhaustion_preserves_existing_free_segment) { + auto old_buffer{make_buffer(2)}; + auto extra_buffer{make_buffer(1)}; + memory_resource::segmented_stack_resource resource; + resource.init(old_buffer.data(), old_buffer.size(), SEGMENT_SIZE); + + // consume the first segment of old_buffer; the second one is left free in the pool, not yet exhausted + ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); + + void* remaining_old_segment{static_cast(segment_payload(old_buffer.data(), 1))}; + + resource.add_extra_memory(extra_buffer.data(), extra_buffer.size()); + + // the newly added buffer's segment is handed out before old_buffer's still-unused leftover segment + ASSERT_EQ(resource.allocate(SEGMENT_SIZE), static_cast(segment_payload(extra_buffer.data(), 0))); + ASSERT_EQ(resource.allocate(SEGMENT_SIZE), remaining_old_segment); + ASSERT_EQ(resource.allocate(1), nullptr); // exhausted +} + +TEST(segmented_stack_resource_test, buffer_list_head_tracks_most_recently_added_buffer) { + auto buffer1{make_buffer(1)}; + auto buffer2{make_buffer(1)}; + + memory_resource::segmented_stack_resource resource; + resource.init(buffer1.data(), buffer1.size(), SEGMENT_SIZE); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer1.data())); + + resource.add_extra_memory(buffer2.data(), buffer2.size()); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer2.data())); + + auto* head{resource.get_buffer_list_head()}; + + ASSERT_EQ(static_cast(head->next), static_cast(buffer1.data())); + + head = head->next; + + ASSERT_EQ(static_cast(head->next), nullptr); +} diff --git a/tests/cpp/runtime/runtime-tests.cmake b/tests/cpp/runtime/runtime-tests.cmake index 731da98fe6..d403b5e385 100644 --- a/tests/cpp/runtime/runtime-tests.cmake +++ b/tests/cpp/runtime/runtime-tests.cmake @@ -14,10 +14,12 @@ prepend(RUNTIME_TESTS_SOURCES ${BASE_DIR}/tests/cpp/runtime/ number-string-comparison.cpp kphp-type-traits-test.cpp msgpack-test.cpp + memory_resource/chunk-pool-resource-test.cpp memory_resource/details/memory_chunk_list-test.cpp memory_resource/details/memory_chunk_tree-test.cpp memory_resource/details/memory_ordered_chunk_list-test.cpp memory_resource/extra-memory-pool-test.cpp + memory_resource/segmented-stack-resource-test.cpp memory_resource/unsynchronized_pool_resource-test.cpp string-list-test.cpp string-test.cpp From 14fef6ec22450e75514a7f218c7ee078f51a4a83 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 1 Oct 2026 20:05:17 +0300 Subject: [PATCH 02/26] move runtime to new task allocator api --- .../components/confdata/state/instance-state.cpp | 4 ++-- .../components/kphp/state/instance-state.cpp | 4 ++-- runtime-light/coroutine/task.h | 2 +- .../component/inter-component-session/client.h | 2 +- runtime-light/stdlib/curl/curl-easy-functions.h | 5 +++-- runtime-light/stdlib/file/resource.h | 4 ++-- runtime-light/stdlib/fork/fork-functions.h | 16 ---------------- runtime-light/stdlib/fork/wait-queue-functions.h | 4 ++-- runtime-light/stdlib/rpc/rpc-api.cpp | 2 +- runtime-light/stdlib/rpc/rpc-queue-functions.h | 4 ++-- runtime-light/stdlib/server/handler-functions.h | 4 ++-- runtime-light/stdlib/server/http-functions.h | 4 ++-- runtime-light/stdlib/time/timer-functions.h | 4 ++-- runtime-light/streams/connection.h | 2 +- 14 files changed, 23 insertions(+), 38 deletions(-) diff --git a/runtime-light/components/confdata/state/instance-state.cpp b/runtime-light/components/confdata/state/instance-state.cpp index d43c7999e8..3abb723ce1 100644 --- a/runtime-light/components/confdata/state/instance-state.cpp +++ b/runtime-light/components/confdata/state/instance-state.cpp @@ -230,7 +230,7 @@ auto InstanceState::erase_if_retired_and_unused(confdata_piece_list::iterator pi } auto InstanceState::init() noexcept -> void { - auto main_task{run()}; + auto main_task{FORK_TASK(run())}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; main_task_async_stack_frame.async_stack_root = std::addressof(m_coroutine_instance_state.coroutine_stack_root); @@ -337,7 +337,7 @@ auto InstanceState::accept_loop() noexcept -> kphp::coro::task<> { } kphp::log::debug("accepted a stream: descriptor -> {}", stream->descriptor()); - if (!m_io_scheduler.start(serve_reader_lease(std::move(*stream)))) [[unlikely]] { + if (!m_io_scheduler.start(FORK_TASK(serve_reader_lease(std::move(*stream))))) [[unlikely]] { kphp::log::warning("failed to serve a confdata reader lease"); } } diff --git a/runtime-light/components/kphp/state/instance-state.cpp b/runtime-light/components/kphp/state/instance-state.cpp index 0698f22329..61e95ba3f8 100644 --- a/runtime-light/components/kphp/state/instance-state.cpp +++ b/runtime-light/components/kphp/state/instance-state.cpp @@ -62,7 +62,7 @@ void InstanceState::init_script_execution() noexcept { kphp::coro::task<> script_task; init_php_scripts_in_each_worker(php_script_mutable_globals_singleton, script_task); - auto main_task{std::invoke( + auto main_task{FORK_TASK(std::invoke( [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { // wrap script with additional check for unhandled exception script_task = std::invoke( @@ -75,7 +75,7 @@ void InstanceState::init_script_execution() noexcept { std::move(script_task)); kphp::log::assertion(co_await f$wait_concurrently(kphp::forks::start(std::move(script_task)))); }, - std::move(script_task))}; + std::move(script_task)))}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; main_task_async_stack_frame.async_stack_root = std::addressof(coroutine_instance_state.coroutine_stack_root); diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 88c31b5b6a..acf5732210 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -253,4 +253,4 @@ struct task { } // namespace kphp::coro -#define FORK_TASK(...) (kphp::coro::detail::memory::task_allocator::get().fork(), ...); +#define FORK_TASK(...) (kphp::coro::detail::memory::task_allocator::get().fork(), __VA_ARGS__) diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 934b958ed9..55755b5025 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -110,7 +110,7 @@ class client final { // The protocol design assumes that interrupting the transfer in the middle of a frame leads to critical error. // Therefore, we need to write the request in a separate coroutine. // This technique prevents integrity violations when this coroutine is cancelled. - kphp::coro::io_scheduler::get().start(process_write(t, qid, payload)); + kphp::coro::io_scheduler::get().start(FORK_TASK(process_write(t, qid, payload))); co_await req_finish_notifier[qid]; kphp::log::assertion(req_status.contains(qid)); diff --git a/runtime-light/stdlib/curl/curl-easy-functions.h b/runtime-light/stdlib/curl/curl-easy-functions.h index 558c06f88d..71aa104e16 100644 --- a/runtime-light/stdlib/curl/curl-easy-functions.h +++ b/runtime-light/stdlib/curl/curl-easy-functions.h @@ -11,6 +11,7 @@ #include #include "runtime-common/core/runtime-core.h" +#include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/curl/curl-context.h" #include "runtime-light/stdlib/curl/curl-state.h" #include "runtime-light/stdlib/curl/defs.h" @@ -473,8 +474,8 @@ inline auto f$curl_exec_concurrently(kphp::web::curl::easy_type easy_id, double timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; - auto sched_res{ - co_await kphp::coro::io_scheduler::get().schedule(kphp::forks::id_managed(kphp::web::simple::perform(kphp::web::simple::transfer{easy_id})), timeout)}; + auto sched_res{co_await kphp::coro::io_scheduler::get().schedule( + FORK_TASK(kphp::forks::id_managed(kphp::web::simple::perform(kphp::web::simple::transfer{easy_id}))), timeout)}; if (!sched_res.has_value()) [[unlikely]] { kphp::web::curl::print_debug( "could not execute curl easy handle concurrently", diff --git a/runtime-light/stdlib/file/resource.h b/runtime-light/stdlib/file/resource.h index 9acbc0e740..04e5fe8b87 100644 --- a/runtime-light/stdlib/file/resource.h +++ b/runtime-light/stdlib/file/resource.h @@ -637,7 +637,7 @@ class socket : public async_resource { socket& operator=(socket&& other) noexcept { if (this != std::addressof(other)) { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(finalizer(std::move(m_stream), std::move(m_buf))); + kphp::coro::io_scheduler::get().spawn(FORK_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); } m_open = std::exchange(other.m_open, false); @@ -649,7 +649,7 @@ class socket : public async_resource { ~socket() override { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(finalizer(std::move(m_stream), std::move(m_buf))); + kphp::coro::io_scheduler::get().spawn(FORK_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); } } diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index ff4133f66a..fd2c4de625 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -59,22 +59,6 @@ auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task -requires(kphp::coro::is_task_function_v) -auto id_managed(F f, Args... args) noexcept -> kphp::coro::task>::awaiter_return_type> { - auto& fork_instance_st{ForkInstanceState::get()}; - const auto saved_fork_id{fork_instance_st.current_id}; - if constexpr (std::is_void_v>::awaiter_return_type>) { - co_await std::invoke(std::move(f), std::move(args)...); - fork_instance_st.current_id = saved_fork_id; - co_return; - } else { - auto value{co_await std::invoke(std::move(f), std::move(args)...)}; - fork_instance_st.current_id = saved_fork_id; - co_return std::move(value); - } -} - template auto start(kphp::coro::task task) noexcept -> int64_t { auto& fork_instance_st{ForkInstanceState::get()}; diff --git a/runtime-light/stdlib/fork/wait-queue-functions.h b/runtime-light/stdlib/fork/wait-queue-functions.h index 1cfa8948c7..196b5be0d8 100644 --- a/runtime-light/stdlib/fork/wait-queue-functions.h +++ b/runtime-light/stdlib/fork/wait-queue-functions.h @@ -56,7 +56,7 @@ inline kphp::coro::task> wait_queue_next(int64_t queue_id timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(wait_queue_next_task(await_set.next()), timeout)}; + auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(FORK_TASK(wait_queue_next_task(await_set.next()), timeout))}; if (!wait_result) { co_return std::nullopt; } @@ -94,7 +94,7 @@ inline void wait_queue_push(int64_t queue_id, int64_t fork_id) noexcept { }}; auto& await_set{(*opt_await_set).get()}; - await_set.push(wait_queue_wrapper_task(std::move(fork_task), fork_id)); + await_set.push(FORK_TASK(wait_queue_wrapper_task(std::move(fork_task), fork_id))); } } // namespace kphp::forks diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp index 5c573dc222..6a6d95dd01 100644 --- a/runtime-light/stdlib/rpc/rpc-api.cpp +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -358,7 +358,7 @@ kphp::rpc::query_info send_request(std::string_view actor, std::optional return {reinterpret_cast(response_exp->buffer()), size}; }}; - auto fetch_task{kphp::rpc::query::response(std::move(q), response_buffer_provider)}; + auto fetch_task{FORK_TASK(kphp::rpc::query::response(std::move(q), response_buffer_provider))}; auto fetch_result{co_await kphp::coro::io_scheduler::get().schedule(std::move(fetch_task))}; if (!fetch_result) [[unlikely]] { response_exp = std::unexpected{fetch_result.error()}; diff --git a/runtime-light/stdlib/rpc/rpc-queue-functions.h b/runtime-light/stdlib/rpc/rpc-queue-functions.h index d120f452fd..0ad3c88ceb 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-functions.h +++ b/runtime-light/stdlib/rpc/rpc-queue-functions.h @@ -43,7 +43,7 @@ inline void rpc_queue_push(int64_t queue_id, int64_t request_id) noexcept { } auto& await_set{(*opt_await_set).get()}; - await_set.push(rpc_queue_wrapper_task(static_cast>(it_awaiter_task->second), request_id)); + await_set.push(FORK_TASK(rpc_queue_wrapper_task(static_cast>(it_awaiter_task->second), request_id))); } inline int64_t rpc_queue_create(std::span request_ids) noexcept { @@ -83,7 +83,7 @@ inline kphp::coro::task> rpc_queue_next(int64_t queue_id, timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(rpc_queue_next_task(await_set.next()), timeout)}; + const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(FORK_TASK(rpc_queue_next_task(await_set.next())), timeout)}; if (!expected_next) { co_return std::nullopt; } diff --git a/runtime-light/stdlib/server/handler-functions.h b/runtime-light/stdlib/server/handler-functions.h index 41c9981b0c..b5a97d14d8 100644 --- a/runtime-light/stdlib/server/handler-functions.h +++ b/runtime-light/stdlib/server/handler-functions.h @@ -19,7 +19,7 @@ void f$register_shutdown_function(F&& f, Args&&... args) noexcept { // it's a lambda coroutine, so: // 1. don't capture anything; // 2. parameters are passed by value. - auto shutdown_function_task{std::invoke( + auto shutdown_function_task{FORK_TASK(std::invoke( [](F f, Args... args) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f), std::move(args)...); @@ -27,7 +27,7 @@ void f$register_shutdown_function(F&& f, Args&&... args) noexcept { std::invoke(std::move(f), std::move(args)...); } }, - std::forward(f), std::forward(args)...)}; + std::forward(f), std::forward(args)...))}; InstanceState::get().shutdown_functions.emplace_back(std::move(shutdown_function_task)); } diff --git a/runtime-light/stdlib/server/http-functions.h b/runtime-light/stdlib/server/http-functions.h index cb4b9bb8ac..a6f086bdc0 100644 --- a/runtime-light/stdlib/server/http-functions.h +++ b/runtime-light/stdlib/server/http-functions.h @@ -75,7 +75,7 @@ bool f$header_register_callback(F&& f) noexcept { return false; } - auto headers_callback_task{std::invoke( + auto headers_callback_task{FORK_TASK(std::invoke( [](F f) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f)); @@ -83,7 +83,7 @@ bool f$header_register_callback(F&& f) noexcept { std::invoke(std::move(f)); } }, - std::forward(f))}; + std::forward(f)))}; http_server_instance_st.headers_registered_callback.emplace(std::move(headers_callback_task)); return true; diff --git a/runtime-light/stdlib/time/timer-functions.h b/runtime-light/stdlib/time/timer-functions.h index e6e0bdbc1f..10e668b9b8 100644 --- a/runtime-light/stdlib/time/timer-functions.h +++ b/runtime-light/stdlib/time/timer-functions.h @@ -24,11 +24,11 @@ void f$set_timer(int64_t timeout_ms, T&& on_timer_callback) noexcept { // TODO choose from: // 1. someone should pop that fork from ForkInstanceState since it will stay there unless we perform f$wait on fork // 2. start timer_task via kphp::coro::io_scheduler::spawn (it won't have distinct fork id) - auto timer_task{std::invoke( + auto timer_task{FORK_TASK(std::invoke( [](std::chrono::milliseconds duration, T on_timer_callback) noexcept -> kphp::coro::task<> { co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(duration)); std::invoke(std::move(on_timer_callback)); }, - std::chrono::milliseconds{timeout_ms}, std::forward(on_timer_callback))}; + std::chrono::milliseconds{timeout_ms}, std::forward(on_timer_callback)))}; kphp::forks::start(std::move(timer_task)); } diff --git a/runtime-light/streams/connection.h b/runtime-light/streams/connection.h index 0a52c083b5..4bd618b9bf 100644 --- a/runtime-light/streams/connection.h +++ b/runtime-light/streams/connection.h @@ -151,7 +151,7 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s }}; m_shared_state.get()->m_unwatch_event.emplace(); - if (!kphp::coro::io_scheduler::get().spawn(watcher(m_stream.descriptor(), m_shared_state, std::forward(h)))) [[unlikely]] { + if (!kphp::coro::io_scheduler::get().spawn(FORK_TASK(watcher(m_stream.descriptor(), m_shared_state, std::forward(h))))) [[unlikely]] { m_shared_state.get()->m_unwatch_event.reset(); return std::unexpected{k2::errno_ebusy}; } From c4f2f4b8c43a03a4b11fdc3d581c225b942bf391 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 1 Oct 2026 20:09:24 +0300 Subject: [PATCH 03/26] change codegen for new task allocator api --- compiler/code-gen/files/init-scripts.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 57b3988697..619bd70360 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -276,7 +276,7 @@ void InitScriptsCpp::compile(CodeGenerator& W) const { W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; if (G->is_output_mode_k2()) { - W << "run = " << FunctionName(main_file_id->main_function) << "$run();" << NL; + W << "run = FORK_TASK(" << FunctionName(main_file_id->main_function) << "$run());" << NL; } else { W << "set_script (" << FunctionName(main_file_id->main_function) << "$run, " << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; } @@ -320,11 +320,7 @@ void ComponentInfoFile::compile(CodeGenerator& W) const { << R"(static std::array extraInfo {ImageInfo::KeyValuePair{.key = "compiler_version", .value = ")" << G->settings().get_version() << "\"}};" << NL << "static ImageInfo imageInfo {\"" << G->settings().k2_component_name.get() << "\"" << "," << (G->is_output_mode_k2_multishot() ? "0" : "1") << "," << std::to_string(std::chrono::duration_cast(G->settings().build_tp.time_since_epoch()).count()) << "," - << "K2_PLATFORM_HEADER_H_VERSION, " - << "{" << php_code_commit_hash << "}," - << "\"" << G->settings().php_code_version.get() << "\"," - << "extraInfo.size()" << "," - << "extraInfo.data()" - << "};" << NL << "return &imageInfo;" << NL << END; + << "K2_PLATFORM_HEADER_H_VERSION, " << "{" << php_code_commit_hash << "}," << "\"" << G->settings().php_code_version.get() << "\"," << "extraInfo.size()" + << "," << "extraInfo.data()" << "};" << NL << "return &imageInfo;" << NL << END; W << CloseFile(); } From f4230cdef860417bd4c1df1d16d7bfa7e208fda9 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Fri, 2 Oct 2026 14:30:45 +0300 Subject: [PATCH 04/26] add new api for kphp::forks::id_managed --- runtime-light/coroutine/task.h | 2 +- .../stdlib/component/component-api.h | 12 ++++--- .../stdlib/crypto/crypto-functions.cpp | 34 +++++++++++++------ .../stdlib/curl/curl-easy-functions.h | 25 +++++++------- .../stdlib/curl/curl-multi-functions.h | 23 +++++++------ .../stdlib/file/file-system-functions.h | 8 ++--- runtime-light/stdlib/fork/fork-functions.h | 23 +++++++++++-- .../stdlib/fork/wait-queue-functions.h | 6 ++-- runtime-light/stdlib/rpc/rpc-api.h | 8 ++--- .../stdlib/rpc/rpc-queue-functions.h | 4 +-- .../stdlib/server/common-functions.h | 2 +- .../stdlib/system/system-functions.h | 8 +++-- runtime-light/stdlib/time/timer-functions.h | 3 +- 13 files changed, 99 insertions(+), 59 deletions(-) diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index acf5732210..9a8e8b62e9 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -177,7 +177,7 @@ struct task { ~task() { if (m_coro) { - auto& task_allocator{m_coro.promise().task_allocator}; + auto& task_allocator{m_coro.promise().m_task_allocator}; auto* prev_stack{task_allocator.exchange_stack(m_coro.promise().m_stack)}; m_coro.destroy(); task_allocator.set_stack(prev_stack); diff --git a/runtime-light/stdlib/component/component-api.h b/runtime-light/stdlib/component/component-api.h index 9719c1fa5c..acd5839408 100644 --- a/runtime-light/stdlib/component/component-api.h +++ b/runtime-light/stdlib/component/component-api.h @@ -119,7 +119,7 @@ inline auto f$component_client_send_request(string name, string request) noexcep auto stream{std::move(*expected_stream)}; auto request_span{std::span{request.c_str(), request.size()}}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::send_request(stream, std::as_bytes(request_span)))}; !expected) [[unlikely]] { + if (auto expected{co_await kphp::forks::id_managed(kphp::component::send_request, stream, std::as_bytes(request_span))}; !expected) [[unlikely]] { co_return class_instance{}; } co_return make_instance(std::move(stream)); @@ -132,7 +132,8 @@ inline auto f$component_client_fetch_response(class_instance q } string response{}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_response(query.get()->stream(), kphp::component::read_ext::append(response)))}; + auto callback{kphp::component::read_ext::append(response)}; + if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_response, query.get()->stream(), std::move(callback))}; !expected) [[unlikely]] { co_return string{}; } @@ -142,7 +143,7 @@ inline auto f$component_client_fetch_response(class_instance q // === component query server interface =========================================================== inline auto f$component_server_accept_query() noexcept -> kphp::coro::task> { - auto opt_stream{co_await kphp::forks::id_managed(kphp::component::stream::accept())}; + auto opt_stream{co_await kphp::forks::id_managed([]() noexcept { return kphp::component::stream::accept(); })}; if (!opt_stream) [[unlikely]] { co_return class_instance{}; } @@ -156,7 +157,8 @@ inline auto f$component_server_fetch_request(class_instance qu } string request{}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_request(query.get()->stream(), kphp::component::read_ext::append(request)))}; + auto callback{kphp::component::read_ext::append(request)}; + if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_request, query.get()->stream(), std::move(callback))}; !expected) [[unlikely]] { co_return string{}; } @@ -171,5 +173,5 @@ inline auto f$component_server_send_response(class_instance qu auto& stream{query.get()->stream()}; auto response_span{std::span{response.c_str(), response.size()}}; - co_await kphp::forks::id_managed(kphp::component::send_response(stream, std::as_bytes(response_span))); + co_await kphp::forks::id_managed(kphp::component::send_response, stream, std::as_bytes(response_span)); } diff --git a/runtime-light/stdlib/crypto/crypto-functions.cpp b/runtime-light/stdlib/crypto/crypto-functions.cpp index 9c83b45f3a..8c682a1045 100644 --- a/runtime-light/stdlib/crypto/crypto-functions.cpp +++ b/runtime-light/stdlib/crypto/crypto-functions.cpp @@ -52,7 +52,8 @@ kphp::coro::task> f$openssl_random_pseudo_bytes(int64_t length) auto stream{*std::move(expected_stream)}; kphp::stl::vector response{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -75,7 +76,8 @@ kphp::coro::task>> f$openssl_x509_parse(string data, bool auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -122,7 +124,8 @@ kphp::coro::task f$openssl_sign(string data, string& signature, string pri auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -149,7 +152,9 @@ kphp::coro::task f$openssl_verify(string data, string signature, string auto stream{*std::move(expected_stream)}; std::array response{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), response))) [[unlikely]] { + if (!co_await kphp::forks::id_managed([](kphp::component::stream& stream, std::span request, + std::span response) noexcept { return kphp::component::query(stream, request, response); }, + stream, tls.view(), response)) [[unlikely]] { co_return 0; } @@ -331,7 +336,8 @@ kphp::coro::task> f$openssl_encrypt(string data, string method, auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -402,7 +408,8 @@ kphp::coro::task> f$openssl_decrypt(string data, string method, auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -428,7 +435,8 @@ kphp::coro::task> f$openssl_pkey_get_public(string key) noexcep auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -457,7 +465,8 @@ kphp::coro::task> f$openssl_pkey_get_private(string key, string auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -486,7 +495,8 @@ kphp::coro::task f$openssl_public_encrypt(string data, string& encrypted_d auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -526,7 +536,8 @@ kphp::coro::task f$openssl_private_decrypt(string data, string& decrypted_ auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } @@ -579,7 +590,8 @@ kphp::coro::task send_and_get_string(tl::storer tls, bool raw_output) no auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { co_return false; } diff --git a/runtime-light/stdlib/curl/curl-easy-functions.h b/runtime-light/stdlib/curl/curl-easy-functions.h index 71aa104e16..81ae6e7912 100644 --- a/runtime-light/stdlib/curl/curl-easy-functions.h +++ b/runtime-light/stdlib/curl/curl-easy-functions.h @@ -24,7 +24,7 @@ #include "runtime-light/stdlib/web-transfer-lib/web-simple-transfer.h" inline auto f$curl_init(string url = string{""}) noexcept -> kphp::coro::task { - auto open_res{co_await kphp::forks::id_managed(kphp::web::simple::open(kphp::web::transfer_backend::CURL))}; + auto open_res{co_await kphp::forks::id_managed(kphp::web::simple::open, kphp::web::transfer_backend::CURL)}; if (!open_res.has_value()) [[unlikely]] { kphp::web::curl::print_warning("could not initialize a new curl easy handle", std::move(open_res.error())); co_return 0; @@ -411,7 +411,7 @@ inline auto f$curl_exec(kphp::web::curl::easy_type easy_id) noexcept -> kphp::co if (!curl_state.easy_ctx.has(easy_id)) { co_return false; } - auto res{co_await kphp::forks::id_managed(kphp::web::simple::perform(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::perform, kphp::web::simple::transfer{easy_id})}; auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; easy_ctx.has_been_executed = true; if (!res.has_value()) [[unlikely]] { @@ -432,7 +432,7 @@ inline auto f$curl_close(kphp::web::curl::easy_type easy_id) noexcept -> kphp::c co_return; } auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::simple::close(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::close, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not close curl easy handle", std::move(res.error())); @@ -445,7 +445,7 @@ inline auto f$curl_reset(kphp::web::curl::easy_type easy_id) noexcept -> kphp::c co_return; } auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::simple::reset(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::reset, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not reset curl easy handle", std::move(res.error())); @@ -475,7 +475,7 @@ inline auto f$curl_exec_concurrently(kphp::web::curl::easy_type easy_id, double auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; auto sched_res{co_await kphp::coro::io_scheduler::get().schedule( - FORK_TASK(kphp::forks::id_managed(kphp::web::simple::perform(kphp::web::simple::transfer{easy_id}))), timeout)}; + FORK_TASK(kphp::forks::id_managed(kphp::web::simple::perform, kphp::web::simple::transfer{easy_id})), timeout)}; if (!sched_res.has_value()) [[unlikely]] { kphp::web::curl::print_debug( "could not execute curl easy handle concurrently", @@ -523,8 +523,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; switch (static_cast(option)) { case kphp::web::curl::CURLINFO::NONE: { - auto res{ - co_await kphp::forks::id_managed(kphp::web::property::get(kphp::web::simple::transfer{easy_id}, std::nullopt, kphp::web::property::get_policy::load))}; + auto res{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, std::nullopt, + kphp::web::property::get_policy::load)}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not get all info options of easy handle", std::move(res.error())); @@ -545,8 +545,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = const auto& image_state{CurlImageState::get()}; if (!easy_ctx.has_been_executed) { const auto url_opt_id{static_cast(kphp::web::curl::CURLOPT::URL)}; - const auto url{co_await kphp::forks::id_managed( - kphp::web::property::get(kphp::web::simple::transfer{easy_id}, url_opt_id, kphp::web::property::get_policy::cached))}; + const auto url{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, url_opt_id, + kphp::web::property::get_policy::cached)}; if (url.has_value()) { const auto& v{(*url).find(url_opt_id)}; kphp::log::assertion(v != (*url).end()); @@ -590,8 +590,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = case kphp::web::curl::CURLINFO::EFFECTIVE_URL: if (!easy_ctx.has_been_executed) { const auto url_opt_id{static_cast(kphp::web::curl::CURLOPT::URL)}; - const auto url{co_await kphp::forks::id_managed( - kphp::web::property::get(kphp::web::simple::transfer{easy_id}, url_opt_id, kphp::web::property::get_policy::cached))}; + const auto url{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, url_opt_id, + kphp::web::property::get_policy::cached)}; if (url.has_value()) { co_return (*url).find(url_opt_id)->second.to_mixed(); } @@ -623,7 +623,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = case kphp::web::curl::CURLINFO::CONDITION_UNMET: case kphp::web::curl::CURLINFO::NUM_CONNECTS: case kphp::web::curl::CURLINFO::HEADER_OUT: { - auto res{co_await kphp::forks::id_managed(kphp::web::property::get(kphp::web::simple::transfer{easy_id}, option, kphp::web::property::get_policy::load))}; + auto res{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, option, + kphp::web::property::get_policy::load)}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not get a specific info of easy handle", std::move(res.error())); diff --git a/runtime-light/stdlib/curl/curl-multi-functions.h b/runtime-light/stdlib/curl/curl-multi-functions.h index c85a7f270f..d764d97308 100644 --- a/runtime-light/stdlib/curl/curl-multi-functions.h +++ b/runtime-light/stdlib/curl/curl-multi-functions.h @@ -23,7 +23,7 @@ #include "runtime-light/stdlib/web-transfer-lib/web-simple-transfer.h" inline auto f$curl_multi_init() noexcept -> kphp::coro::task { - auto open_res{co_await kphp::forks::id_managed(kphp::web::composite::open(kphp::web::transfer_backend::CURL))}; + auto open_res{co_await kphp::forks::id_managed(kphp::web::composite::open, kphp::web::transfer_backend::CURL)}; if (!open_res.has_value()) [[unlikely]] { kphp::web::curl::print_warning("could not initialize a new curl multi handle", std::move(open_res.error())); co_return 0; @@ -45,7 +45,7 @@ inline auto f$curl_multi_add_handle(kphp::web::curl::multi_type multi_id, kphp:: auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; easy_ctx.errors_reset(); - auto res{co_await kphp::forks::id_managed(kphp::web::composite::add(kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::add, kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not add a curl easy handler into multi handle", std::move(res.error())); @@ -65,7 +65,7 @@ inline auto f$curl_multi_remove_handle(kphp::web::curl::multi_type multi_id, if (!curl_state.easy_ctx.has(easy_id)) [[unlikely]] { co_return false; } - auto res{co_await kphp::forks::id_managed(kphp::web::composite::remove(kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::remove, kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not remove a curl easy handler from multi handle", std::move(res.error())); @@ -127,7 +127,7 @@ inline auto f$curl_multi_exec(kphp::web::curl::multi_type multi_id, int64_t& sti if (!curl_state.multi_ctx.has(multi_id)) [[unlikely]] { co_return false; } - auto res{co_await kphp::forks::id_managed(kphp::web::composite::perform(kphp::web::composite::transfer{multi_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::perform, kphp::web::composite::transfer{multi_id})}; auto& multi_ctx{curl_state.multi_ctx.get_or_init(multi_id)}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); @@ -146,7 +146,7 @@ inline auto f$curl_multi_getcontent(kphp::web::curl::easy_type easy_id) noexcept } auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; if (easy_ctx.return_transfer) { - auto res{co_await kphp::forks::id_managed(kphp::web::simple::get_response(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::get_response, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not get response of curl easy handle", std::move(res.error())); @@ -163,7 +163,7 @@ inline auto f$curl_multi_close(kphp::web::curl::multi_type multi_id) noexcept -> co_return; } auto& multi_ctx{curl_state.multi_ctx.get_or_init(multi_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::composite::close(kphp::web::composite::transfer{multi_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::close, kphp::web::composite::transfer{multi_id})}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not close curl multi handle", std::move(res.error())); @@ -223,8 +223,11 @@ inline auto f$curl_multi_select(kphp::web::curl::multi_type multi_id, double tim co_return false; } auto& multi_ctx{curl_state.multi_ctx.get_or_init(multi_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::composite::wait_updates( - kphp::web::composite::transfer{multi_id}, std::chrono::duration_cast(std::chrono::duration{timeout})))}; + auto res{co_await kphp::forks::id_managed( + [](kphp::web::composite::transfer multi_id, std::chrono::seconds timeout) noexcept { + return kphp::web::composite::wait_updates(kphp::web::composite::transfer{multi_id}, timeout); + }, + kphp::web::composite::transfer{multi_id}, std::chrono::duration_cast(std::chrono::duration{timeout}))}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not select curl multi handle", std::move(res.error())); @@ -243,8 +246,8 @@ inline auto f$curl_multi_info_read(kphp::web::curl::multi_type multi_id, constexpr auto CURL_MULTI_INFO_READ_OPTION = 0; - auto props{co_await kphp::forks::id_managed( - kphp::web::property::get(kphp::web::composite::transfer{multi_id}, CURL_MULTI_INFO_READ_OPTION, kphp::web::property::get_policy::load))}; + auto props{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::composite::transfer{multi_id}, + CURL_MULTI_INFO_READ_OPTION, kphp::web::property::get_policy::load)}; if (!props.has_value()) [[unlikely]] { multi_ctx.set_errno(props.error().code, props.error().description); kphp::web::curl::print_warning("could not get info message of multi handle", std::move(props.error())); diff --git a/runtime-light/stdlib/file/file-system-functions.h b/runtime-light/stdlib/file/file-system-functions.h index 8741257e5f..d99fbfd5f3 100644 --- a/runtime-light/stdlib/file/file-system-functions.h +++ b/runtime-light/stdlib/file/file-system-functions.h @@ -110,7 +110,7 @@ inline kphp::coro::task f$fclose(resource stream) noexcept { if (auto sync_resource{from_mixed>(stream, {})}; !sync_resource.is_null()) { co_return sync_resource.get()->close(); } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - co_return co_await kphp::forks::id_managed(async_resource.get()->close()); + co_return co_await kphp::forks::id_managed(&kphp::fs::async_resource::close, async_resource.get()); } kphp::log::warning("unexpected resource in fclose -> {}", stream.to_string().c_str()); @@ -133,7 +133,7 @@ inline kphp::coro::task> f$fwrite(resource stream, string data auto expected{sync_resource.get()->write(std::as_bytes(data_span))}; co_return expected ? Optional{static_cast(*std::move(expected))} : Optional{false}; } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - auto expected{co_await kphp::forks::id_managed(async_resource.get()->write(std::as_bytes(data_span)))}; + auto expected{co_await kphp::forks::id_managed(&kphp::fs::async_resource::write, async_resource.get(), std::as_bytes(data_span))}; co_return expected ? Optional{static_cast(*std::move(expected))} : Optional{false}; } @@ -152,7 +152,7 @@ inline kphp::coro::task> f$fread(resource stream, int64_t lengt if (auto sync_resource{from_mixed>(stream, {})}; !sync_resource.is_null()) { expected = sync_resource.get()->read(std::as_writable_bytes(buf)); } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - expected = co_await kphp::forks::id_managed(async_resource.get()->read(std::as_writable_bytes(buf))); + expected = co_await kphp::forks::id_managed(&kphp::fs::async_resource::read, async_resource.get(), std::as_writable_bytes(buf)); } else { kphp::log::warning("unexpected resource in fread -> {}", stream.to_string().c_str()); co_return false; @@ -168,7 +168,7 @@ inline kphp::coro::task f$fflush(resource stream) noexcept { if (auto sync_resource{from_mixed>(stream, {})}; !sync_resource.is_null()) { co_return sync_resource.get()->flush(); } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - co_return co_await kphp::forks::id_managed(async_resource.get()->flush()); + co_return co_await kphp::forks::id_managed(&kphp::fs::async_resource::flush, async_resource.get()); } kphp::log::warning("unexpected resource in fflush -> {}", stream.to_string().c_str()); diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index fd2c4de625..2ed12e1888 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -59,6 +59,23 @@ auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task +requires(kphp::coro::is_task_function_v) +auto id_managed(F f, + Args&&... args) noexcept -> kphp::coro::task>::awaiter_return_type> { + auto& fork_instance_st{ForkInstanceState::get()}; + const auto saved_fork_id{fork_instance_st.current_id}; + if constexpr (std::is_void_v>::awaiter_return_type>) { + co_await std::invoke(std::move(f), std::forward(args)...); + fork_instance_st.current_id = saved_fork_id; + co_return; + } else { + auto value{co_await std::invoke(std::move(f), std::forward(args)...)}; + fork_instance_st.current_id = saved_fork_id; + co_return std::move(value); + } +} + template auto start(kphp::coro::task task) noexcept -> int64_t { auto& fork_instance_st{ForkInstanceState::get()}; @@ -127,7 +144,8 @@ auto wait(int64_t fork_id, duration_type timeout) noexcept -> kphp::coro::task requires(is_optional::value || std::same_as || is_class_instance::value) kphp::coro::task f$wait(int64_t fork_id, double timeout = -1.0) noexcept { - auto opt_result{co_await kphp::forks::id_managed(kphp::forks::wait(fork_id, std::chrono::duration{timeout}))}; + auto opt_result{ + co_await kphp::forks::id_managed(kphp::forks::wait>, fork_id, std::chrono::duration{timeout})}; co_return opt_result ? return_type{*std::move(opt_result)} : return_type{}; } @@ -196,7 +214,8 @@ inline kphp::coro::task<> f$sched_yield() noexcept { } inline kphp::coro::task<> f$sched_yield_sleep(double duration) noexcept { - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(std::chrono::duration{duration})); + co_await kphp::forks::id_managed([](std::chrono::duration duration) noexcept { return kphp::coro::io_scheduler::get().schedule(duration); }, + std::chrono::duration{duration}); } // ================================================================================================ diff --git a/runtime-light/stdlib/fork/wait-queue-functions.h b/runtime-light/stdlib/fork/wait-queue-functions.h index 196b5be0d8..1ffb860803 100644 --- a/runtime-light/stdlib/fork/wait-queue-functions.h +++ b/runtime-light/stdlib/fork/wait-queue-functions.h @@ -56,7 +56,7 @@ inline kphp::coro::task> wait_queue_next(int64_t queue_id timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(FORK_TASK(wait_queue_next_task(await_set.next()), timeout))}; + auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(FORK_TASK(wait_queue_next_task(await_set.next())), timeout)}; if (!wait_result) { co_return std::nullopt; } @@ -118,8 +118,8 @@ inline bool f$wait_queue_empty(int64_t queue_id) noexcept { } inline kphp::coro::task> f$wait_queue_next(int64_t queue_id, double timeout = -1.0) noexcept { - auto opt_result{co_await kphp::forks::id_managed( - kphp::forks::wait_queue_next(queue_id, std::chrono::duration_cast(std::chrono::duration{timeout})))}; + auto opt_result{co_await kphp::forks::id_managed(kphp::forks::wait_queue_next, queue_id, + std::chrono::duration_cast(std::chrono::duration{timeout}))}; co_return opt_result ? *opt_result : Optional{false}; } diff --git a/runtime-light/stdlib/rpc/rpc-api.h b/runtime-light/stdlib/rpc/rpc-api.h index 4e162f775b..db17c02bb1 100644 --- a/runtime-light/stdlib/rpc/rpc-api.h +++ b/runtime-light/stdlib/rpc/rpc-api.h @@ -277,7 +277,7 @@ inline kphp::coro::task f$store_error(int64_t error_code, string error_msg tl::storer tls{rpc_response.footprint()}; rpc_response.store(tls); - if (auto expected{co_await kphp::forks::id_managed(kphp::rpc::send_response(tls.view()))}; !expected) [[unlikely]] { + if (auto expected{co_await kphp::forks::id_managed(kphp::rpc::send_response, tls.view())}; !expected) [[unlikely]] { kphp::log::warning("can't store RPC error: {}", expected.error()); } kphp::log::error("store_error called. error_code: {}, error_msg: {}", error_code, error_msg.c_str()); @@ -300,7 +300,7 @@ inline kphp::coro::task<> f$rpc_server_store_response(class_instance f$rpc_send_requests(const string& actor, const array>> f$rpc_fetch_responses(array query_ids) noexcept { array> res{query_ids.size()}; for (const auto& it : std::as_const(query_ids)) { - res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::rpc_tl_query_result_one_impl(it.get_value()))); + res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::rpc_tl_query_result_one_impl, it.get_value())); } co_return std::move(res); } @@ -392,7 +392,7 @@ requires std::default_initializable kphp::coro::task>> f$rpc_fetch_typed_responses(array query_ids) noexcept { array> res{query_ids.size()}; for (const auto& it : std::as_const(query_ids)) { - res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::typed_rpc_tl_query_result_one_impl(it.get_value(), error_factory_type{}))); + res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::typed_rpc_tl_query_result_one_impl, it.get_value(), error_factory_type{})); } co_return std::move(res); } diff --git a/runtime-light/stdlib/rpc/rpc-queue-functions.h b/runtime-light/stdlib/rpc/rpc-queue-functions.h index 0ad3c88ceb..27eb74307d 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-functions.h +++ b/runtime-light/stdlib/rpc/rpc-queue-functions.h @@ -136,7 +136,7 @@ inline bool f$rpc_queue_empty(int64_t queue_id) noexcept { } inline kphp::coro::task> f$rpc_queue_next(int64_t queue_id, double timeout = -1) noexcept { - auto opt_result{co_await kphp::forks::id_managed( - kphp::rpc::rpc_queue_next(queue_id, std::chrono::duration_cast(std::chrono::duration{timeout})))}; + auto opt_result{co_await kphp::forks::id_managed(kphp::rpc::rpc_queue_next, queue_id, + std::chrono::duration_cast(std::chrono::duration{timeout}))}; co_return opt_result ? *opt_result : Optional{false}; } diff --git a/runtime-light/stdlib/server/common-functions.h b/runtime-light/stdlib/server/common-functions.h index f622501743..d9d8573da8 100644 --- a/runtime-light/stdlib/server/common-functions.h +++ b/runtime-light/stdlib/server/common-functions.h @@ -31,7 +31,7 @@ inline auto f$ignore_user_abort(Optional enable) noexcept -> kphp::coro::t http_server_instance_st.connection->decrease_ignore_abort_level(); if (http_server_instance_st.connection->get_ignore_abort_level() == 0 && http_server_instance_st.connection->is_aborted()) { - co_await kphp::forks::id_managed(kphp::system::exit(1)); + co_await kphp::forks::id_managed(kphp::system::exit, 1); } co_return prev; } diff --git a/runtime-light/stdlib/system/system-functions.h b/runtime-light/stdlib/system/system-functions.h index 09cad7370a..65d0661a47 100644 --- a/runtime-light/stdlib/system/system-functions.h +++ b/runtime-light/stdlib/system/system-functions.h @@ -95,7 +95,7 @@ inline kphp::coro::task<> f$exit(mixed v = 0) noexcept { // TODO: make it synchr } else { exit_code = 1; } - co_await kphp::forks::id_managed(kphp::system::exit(static_cast(exit_code))); + co_await kphp::forks::id_managed(kphp::system::exit, static_cast(exit_code)); } inline kphp::coro::task<> f$die(mixed v = 0) noexcept { @@ -221,7 +221,8 @@ inline kphp::coro::task<> f$usleep(int64_t microseconds) noexcept { kphp::log::warning("value of microseconds ({}) must be positive", microseconds); co_return; } - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(std::chrono::microseconds{microseconds})); + co_await kphp::forks::id_managed([](std::chrono::microseconds microseconds) noexcept { return kphp::coro::io_scheduler::get().schedule(microseconds); }, + std::chrono::microseconds{microseconds}); } inline kphp::coro::task<> f$sleep(int64_t seconds) noexcept { @@ -230,7 +231,8 @@ inline kphp::coro::task<> f$sleep(int64_t seconds) noexcept { co_return; } - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(std::chrono::seconds{seconds})); + co_await kphp::forks::id_managed([](std::chrono::seconds seconds) noexcept { return kphp::coro::io_scheduler::get().schedule(seconds); }, + std::chrono::seconds{seconds}); co_return; } diff --git a/runtime-light/stdlib/time/timer-functions.h b/runtime-light/stdlib/time/timer-functions.h index 10e668b9b8..00259c6b74 100644 --- a/runtime-light/stdlib/time/timer-functions.h +++ b/runtime-light/stdlib/time/timer-functions.h @@ -26,7 +26,8 @@ void f$set_timer(int64_t timeout_ms, T&& on_timer_callback) noexcept { // 2. start timer_task via kphp::coro::io_scheduler::spawn (it won't have distinct fork id) auto timer_task{FORK_TASK(std::invoke( [](std::chrono::milliseconds duration, T on_timer_callback) noexcept -> kphp::coro::task<> { - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(duration)); + co_await kphp::forks::id_managed([](std::chrono::milliseconds duration) noexcept { return kphp::coro::io_scheduler::get().schedule(duration); }, + duration); std::invoke(std::move(on_timer_callback)); }, std::chrono::milliseconds{timeout_ms}, std::forward(on_timer_callback)))}; From ce8c4ede47d91bf355b336ae5d048d321ff4b03b Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Fri, 2 Oct 2026 18:41:36 +0300 Subject: [PATCH 05/26] add allocator switching --- .../confdata/state/instance-state.cpp | 2 +- .../components/kphp/state/instance-state.cpp | 4 +- runtime-light/coroutine/async-stack.h | 14 ----- runtime-light/coroutine/control-functions.h | 45 +++++++++++++++ runtime-light/coroutine/detail/await-set.h | 34 +++++++---- runtime-light/coroutine/detail/poll-info.h | 14 +++-- runtime-light/coroutine/detail/when-all.h | 45 ++++++++++++--- runtime-light/coroutine/detail/when-any.h | 46 +++++++++++---- runtime-light/coroutine/event.h | 11 +++- runtime-light/coroutine/io-scheduler.h | 56 ++++++++++++------- runtime-light/coroutine/shared-task.h | 45 ++++++++++----- .../coroutine/task-allocator-guard.h | 31 ++++++++++ runtime-light/coroutine/task.h | 1 + .../inter-component-session/client.h | 2 +- runtime-light/streams/connection.h | 2 +- 15 files changed, 265 insertions(+), 87 deletions(-) create mode 100644 runtime-light/coroutine/control-functions.h create mode 100644 runtime-light/coroutine/task-allocator-guard.h diff --git a/runtime-light/components/confdata/state/instance-state.cpp b/runtime-light/components/confdata/state/instance-state.cpp index 3abb723ce1..63bd32e98d 100644 --- a/runtime-light/components/confdata/state/instance-state.cpp +++ b/runtime-light/components/confdata/state/instance-state.cpp @@ -240,7 +240,7 @@ auto InstanceState::init() noexcept -> void { } auto InstanceState::run() noexcept -> kphp::coro::task<> { - co_await kphp::coro::when_all(service_loop(), accept_loop(), metrics_loop()); // all never return + co_await kphp::coro::when_all(FORK_TASK(service_loop()), FORK_TASK(accept_loop()), FORK_TASK(metrics_loop())); // all never return kphp::log::assertion(false); } diff --git a/runtime-light/components/kphp/state/instance-state.cpp b/runtime-light/components/kphp/state/instance-state.cpp index 61e95ba3f8..8d8bda794a 100644 --- a/runtime-light/components/kphp/state/instance-state.cpp +++ b/runtime-light/components/kphp/state/instance-state.cpp @@ -65,14 +65,14 @@ void InstanceState::init_script_execution() noexcept { auto main_task{FORK_TASK(std::invoke( [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { // wrap script with additional check for unhandled exception - script_task = std::invoke( + script_task = FORK_TASK(std::invoke( [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { co_await script_task; if (auto exception{std::move(ForkInstanceState::get().current_info().get().thrown_exception)}; !exception.is_null()) [[unlikely]] { kphp::log::error("unhandled exception {}", std::move(exception)); } }, - std::move(script_task)); + std::move(script_task))); kphp::log::assertion(co_await f$wait_concurrently(kphp::forks::start(std::move(script_task)))); }, std::move(script_task)))}; diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 85ab351a5f..d9b842c492 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -4,9 +4,6 @@ #pragma once -#include -#include - /** * This header defines the data structures used to represent a coroutine asynchronous stack. * @@ -83,17 +80,6 @@ struct async_stack_root { stack_frame* stop_sync_stack_frame{}; }; -/** - * The `resume` function is responsible for storing the current synchronous stack frame - * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows - * capturing one of the stack frames in the synchronous stack trace. - */ -inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root) noexcept { - auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; - handle.resume(); - stack_root.stop_sync_stack_frame = previous_stack_frame; -} - /** * The async_stack_element class facilitates working with asynchronous traces in templated code. * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. diff --git a/runtime-light/coroutine/control-functions.h b/runtime-light/coroutine/control-functions.h new file mode 100644 index 0000000000..d7529089a5 --- /dev/null +++ b/runtime-light/coroutine/control-functions.h @@ -0,0 +1,45 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" + +namespace kphp::coro { + +/** + * The `resume` function is responsible for storing the current synchronous stack frame + * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows + * capturing one of the stack frames in the synchronous stack trace. + * All calls to resume() method of handle must be made with overload of this function. + */ +inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { + kphp::coro::task_allocator_guard guard{task_allocator}; + auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; + handle.resume(); + stack_root.stop_sync_stack_frame = previous_stack_frame; +} + +/* + * All calls to resume() method of handle must be made with overload of this function. + */ +inline void resume(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { + kphp::coro::task_allocator_guard guard{task_allocator}; + handle.resume(); +} + +/* + * All calls to destroy() method of handle must be made with this function (except for task destructor). + */ +inline void destroy(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { + kphp::coro::task_allocator_guard guard{task_allocator}; + handle.destroy(); +} + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index b45f177559..bf0bf60d5a 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -9,10 +9,14 @@ #include #include #include +#include #include "common/containers/intrusive-list.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -41,6 +45,7 @@ class await_broker { vk::intrusive::list>> m_awaiters; await_set_ready_task_element* m_ready_tasks{}; size_t m_tasks_count{0}; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; public: await_broker() noexcept = default; @@ -82,7 +87,7 @@ class await_broker { * (in the future invariant [1] may not work). */ m_awaiters.pop_front(); - coroutine.resume(); + kphp::coro::resume(coroutine, m_task_allocator); } } @@ -116,7 +121,7 @@ class await_broker { */ m_tasks_storage.erase(task_iterator); --m_tasks_count; - typed_handle.destroy(); + kphp::coro::destroy(typed_handle, m_task_allocator); return result_t{std::move(result)}; } @@ -125,14 +130,14 @@ class await_broker { m_ready_tasks = nullptr; detach_all(); while (!m_tasks_storage.empty()) { - auto coroutine{m_tasks_storage.front()}; + auto coroutine{std::coroutine_handle::promise_type>::from_address(m_tasks_storage.front().address())}; /* * We can remove pop_front() here, because list node will be destroyed after destruction of coroutine frame and in its * destructor will call unlink() [1]. But we left pop_front() for better readability and safety * (in the future invariant [1] may not work). */ m_tasks_storage.pop_front(); - coroutine.destroy(); + kphp::coro::destroy(coroutine, m_task_allocator); } m_tasks_count = 0; @@ -150,7 +155,7 @@ class await_broker { * (in the future invariant [1] may not work). */ awaiters.pop_front(); - coroutine.resume(); + kphp::coro::resume(coroutine, m_task_allocator); } } @@ -158,6 +163,10 @@ class await_broker { return m_tasks_count; } + kphp::coro::detail::memory::task_allocator& task_allocator() noexcept { + return m_task_allocator; + } + ~await_broker() { abort_all(); } @@ -192,7 +201,10 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { } auto final_suspend() const noexcept { - struct completion_notifier { + struct completion_notifier : private kphp::coro::task_allocator_guard { + + explicit completion_notifier(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard(task_allocator) {} constexpr auto await_ready() const noexcept { return false; @@ -207,7 +219,8 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return completion_notifier{}; + + return completion_notifier{m_await_broker->get().task_allocator()}; } void unhandled_exception() const noexcept { @@ -231,7 +244,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_await_broker->get().task_allocator()); } }; @@ -314,14 +327,15 @@ class await_set_awaitable { private: await_broker& m_await_broker; - class awaiter { + class awaiter : private kphp::coro::task_allocator_guard { vk::intrusive::list_node> m_awaiting_coroutine_node; await_broker& m_await_broker; kphp::coro::async_stack_frame* caller_frame{}; public: explicit awaiter(await_broker& await_broker) noexcept - : m_await_broker(await_broker) {} + : kphp::coro::task_allocator_guard(await_broker.task_allocator()), + m_await_broker(await_broker) {} awaiter(awaiter&& other) noexcept = delete; awaiter(const awaiter& other) = delete; diff --git a/runtime-light/coroutine/detail/poll-info.h b/runtime-light/coroutine/detail/poll-info.h index 30c153a629..9b2c68edc4 100644 --- a/runtime-light/coroutine/detail/poll-info.h +++ b/runtime-light/coroutine/detail/poll-info.h @@ -11,7 +11,9 @@ #include "common/containers/intrusive-list.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/poll.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/k2-platform/k2-api.h" namespace kphp::coro::detail { @@ -39,9 +41,12 @@ struct poll_info { kphp::coro::poll_status m_poll_status{kphp::coro::poll_status::error}; kphp::coro::poll_op m_poll_op; - poll_info(k2::descriptor descriptor, kphp::coro::poll_op poll_op) noexcept + kphp::coro::detail::memory::task_allocator& m_task_allocator; + + poll_info(k2::descriptor descriptor, kphp::coro::poll_op poll_op, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept : m_descriptor(descriptor), - m_poll_op(poll_op) {} + m_poll_op(poll_op), + m_task_allocator(task_allocator) {} ~poll_info() = default; @@ -51,11 +56,12 @@ struct poll_info { poll_info& operator=(detail::poll_info&&) = delete; auto operator co_await() noexcept { - struct poll_awaiter { + struct poll_awaiter : private kphp::coro::task_allocator_guard { detail::poll_info& m_poll_info; explicit poll_awaiter(detail::poll_info& poll_info) noexcept - : m_poll_info(poll_info) {} + : kphp::coro::task_allocator_guard(poll_info.m_task_allocator), + m_poll_info(poll_info) {} constexpr auto await_ready() const noexcept -> bool { return false; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index 186ba2793c..4af87fdd20 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -16,7 +16,10 @@ #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -26,6 +29,7 @@ namespace kphp::coro::detail::when_all { class when_all_latch { size_t m_count{1}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; public: explicit when_all_latch(size_t count) noexcept @@ -33,7 +37,8 @@ class when_all_latch { when_all_latch(when_all_latch&& other) noexcept : m_count(std::exchange(other.m_count, 1)), - m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})) {} + m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})), + m_task_allocator(other.m_task_allocator) {} auto operator=(when_all_latch&& other) noexcept -> when_all_latch& { if (this != std::addressof(other)) { @@ -59,9 +64,13 @@ class when_all_latch { auto notify_awaitable_completed() noexcept -> void { if (--m_count == 1 && m_awaiting_coroutine != nullptr) { - m_awaiting_coroutine.resume(); + kphp::coro::resume(m_awaiting_coroutine, m_task_allocator); } } + + auto task_allocator() noexcept -> kphp::coro::detail::memory::task_allocator& { + return m_task_allocator; + } }; template @@ -89,13 +98,14 @@ class when_all_ready_awaitable> { when_all_latch m_latch; std::tuple m_tasks; - struct awaiter { + struct awaiter : private kphp::coro::task_allocator_guard { bool m_started{}; when_all_ready_awaitable& m_awaitable; kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; explicit awaiter(when_all_ready_awaitable& awaitable) noexcept - : m_awaitable(awaitable) {} + : kphp::coro::task_allocator_guard(awaitable.m_latch.task_allocator()), + m_awaitable(awaitable) {} auto await_ready() noexcept -> bool { kphp::log::assertion(!std::exchange(m_started, true)); // to make sure it's not co_awaited more than once @@ -120,7 +130,14 @@ class when_all_ready_awaitable> { kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); m_caller_async_stack_frame->async_stack_root->top_async_stack_frame = m_caller_async_stack_frame; } - return std::apply([](task_types&&... tasks) noexcept { return std::make_tuple(std::move(tasks).result()...); }, std::move(m_awaitable.m_tasks)); + + return std::apply( + [&latch = m_awaitable.m_latch](task_types&&... tasks) noexcept { + auto result{std::make_tuple(std::move(tasks).result()...)}; + (tasks.reset(latch.task_allocator()), ...); + return std::move(result); + }, + std::move(m_awaitable.m_tasks)); } }; @@ -170,7 +187,10 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { } auto final_suspend() const noexcept { - struct completion_notifier { + struct completion_notifier : private kphp::coro::task_allocator_guard { + explicit completion_notifier(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard(task_allocator) {} + constexpr auto await_ready() const noexcept { return false; } @@ -181,7 +201,8 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return completion_notifier{}; + + return completion_notifier{m_latch->task_allocator()}; } auto unhandled_exception() const noexcept -> void { @@ -198,7 +219,7 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), latch.task_allocator()); } }; @@ -270,7 +291,7 @@ class when_all_task { ~when_all_task() { if (m_coroutine != nullptr) { - m_coroutine.destroy(); + kphp::coro::destroy(m_coroutine, kphp::coro::detail::memory::task_allocator::get()); } } @@ -281,6 +302,12 @@ class when_all_task { auto result() && noexcept { return m_coroutine.promise().result(); } + + auto reset(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept -> void { + if (m_coroutine != nullptr) { + kphp::coro::destroy(std::exchange(m_coroutine, nullptr), task_allocator); + } + } }; template diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 8a90b1961f..75807a3fea 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -13,8 +13,12 @@ #include #include +#include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/metaprogramming/type-functions.h" @@ -25,13 +29,15 @@ namespace kphp::coro::detail::when_any { class when_any_latch { bool m_toggled{}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; public: when_any_latch() noexcept = default; when_any_latch(when_any_latch&& other) noexcept : m_toggled(std::exchange(other.m_toggled, false)), - m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})) {} + m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})), + m_task_allocator(other.m_task_allocator) {} auto operator=(when_any_latch&& other) noexcept -> when_any_latch& { if (this != std::addressof(other)) { @@ -58,9 +64,13 @@ class when_any_latch { auto notify_awaitable_completed() noexcept -> void { m_toggled = true; if (m_awaiting_coroutine != nullptr) { - m_awaiting_coroutine.resume(); + kphp::coro::resume(m_awaiting_coroutine, m_task_allocator); } } + + auto task_allocator() noexcept -> kphp::coro::detail::memory::task_allocator& { + return m_task_allocator; + } }; template @@ -89,13 +99,14 @@ class when_any_ready_awaitable> { std::tuple m_tasks; std::optional> m_result; - struct awaiter { + struct awaiter : private kphp::coro::task_allocator_guard { bool m_started{}; when_any_ready_awaitable& m_awaitable; kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; explicit awaiter(when_any_ready_awaitable& awaitable) noexcept - : m_awaitable(awaitable) {} + : kphp::coro::task_allocator_guard(awaitable.m_latch.task_allocator()), + m_awaitable(awaitable) {} auto await_ready() noexcept -> bool { kphp::log::assertion(!std::exchange(m_started, true)); // to make sure it's not co_awaited more than once @@ -129,9 +140,14 @@ class when_any_ready_awaitable> { result_variant_type{std::in_place_index()>, *std::move(task_result)}; } }}; - std::apply([&task_result_processor](auto&&... tasks) noexcept { (std::invoke(task_result_processor, std::forward(tasks)), ...); }, - std::move(m_awaitable.m_tasks)); + std::apply( + [&task_result_processor, &latch = m_awaitable.m_latch](auto&&... tasks) noexcept { + (std::invoke(task_result_processor, std::forward(tasks)), ...); + (tasks.reset(latch.task_allocator()), ...); + }, + std::move(m_awaitable.m_tasks)); kphp::log::assertion(m_awaitable.m_result.has_value()); + return std::move(*m_awaitable.m_result); } }; @@ -180,7 +196,10 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { } auto final_suspend() const noexcept { - struct completion_notifier { + struct completion_notifier : private kphp::coro::task_allocator_guard { + explicit completion_notifier(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard(task_allocator) {} + constexpr auto await_ready() const noexcept { return false; } @@ -191,7 +210,8 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return completion_notifier{}; + + return completion_notifier{m_latch->task_allocator()}; } auto unhandled_exception() const noexcept -> void { @@ -208,7 +228,7 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_latch->task_allocator()); } }; @@ -291,7 +311,7 @@ class when_any_task { ~when_any_task() { if (m_coroutine != nullptr) { - m_coroutine.destroy(); + kphp::coro::destroy(m_coroutine, kphp::coro::detail::memory::task_allocator::get()); } } @@ -302,6 +322,12 @@ class when_any_task { auto result() && noexcept { return m_coroutine.promise().result(); } + + auto reset(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept -> void { + if (m_coroutine != nullptr) { + kphp::coro::destroy(std::exchange(m_coroutine, nullptr), task_allocator); + } + } }; template diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index c8d7a98479..dbd26ee6a8 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -15,7 +15,10 @@ #include "common/wrappers/overloaded.h" #include "runtime-common/core/allocator/script-allocator-managed.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { @@ -26,13 +29,14 @@ class event { // 2) non empty list => linked list of coroutines waiting for the event to trigger // 3) empty list => the event is triggered and all coroutines are resumed std::variant>>> m_state; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; auto set() noexcept -> void; auto unset() noexcept -> void; auto is_set() const noexcept -> bool; }; - struct awaiter { + struct awaiter : private kphp::coro::task_allocator_guard { vk::intrusive::list_node> m_awaiting_coroutine_node; bool m_suspended{}; event_controller& m_controller; @@ -40,7 +44,8 @@ class event { kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; explicit awaiter(event_controller& event_controller) noexcept - : m_controller(event_controller), + : kphp::coro::task_allocator_guard(event_controller.m_task_allocator), + m_controller(event_controller), m_async_stack_root(kphp::coro::instance_state::get().coroutine_stack_root) {} awaiter(const awaiter&) = delete; @@ -131,7 +136,7 @@ inline auto event::event_controller::set() noexcept -> void { * (in the future invariant [1] may not work). */ awaiters.pop_front(); - coroutine.resume(); + kphp::coro::resume(coroutine, m_task_allocator); } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 2cc9f01cf8..bf9d084789 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -26,11 +26,14 @@ #include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/detail/poll-info.h" #include "runtime-light/coroutine/detail/task-self-deleting.h" #include "runtime-light/coroutine/detail/timer-handle.h" #include "runtime-light/coroutine/poll.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/when-any.h" @@ -436,7 +439,7 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { */ scheduled_coroutines.pop_front(); kphp::log::assertion(static_cast(coroutine)); - kphp::coro::resume(coroutine, m_coro_instance_state.coroutine_stack_root); + kphp::coro::resume(coroutine, m_coro_instance_state.coroutine_stack_root, m_coro_instance_state.task_allocator); } } @@ -461,7 +464,7 @@ auto io_scheduler::start(coroutine_type coroutine) noexcept -> bool { if (!handle || handle.done()) [[unlikely]] { return false; } - kphp::coro::resume(handle, m_coro_instance_state.coroutine_stack_root); + kphp::coro::resume(handle, m_coro_instance_state.coroutine_stack_root, m_coro_instance_state.task_allocator); return true; } @@ -505,21 +508,36 @@ inline auto io_scheduler::schedule() noexcept { m_schedule_pos); } - constexpr auto await_ready() const noexcept -> bool { - return false; - } - - auto await_suspend(std::coroutine_handle<> coroutine) noexcept -> void { - m_awaiting_coroutine_node.value() = coroutine; - m_scheduler.m_scheduled_coroutines.push_back(m_awaiting_coroutine_node); - m_schedule_pos = std::prev(m_scheduler.m_scheduled_coroutines.end()); - } - - auto await_resume() noexcept -> void { - m_schedule_pos = std::monostate{}; - m_scheduler.m_coro_instance_state.coroutine_stack_root.top_async_stack_frame = m_async_stack_frame; + auto operator co_await() noexcept { + struct awaiter : private kphp::coro::task_allocator_guard { + private: + schedule_operation& m_schedule_operation; + + public: + explicit awaiter(schedule_operation& schedule_operation, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard{task_allocator}, + m_schedule_operation{schedule_operation} {} + + constexpr auto await_ready() const noexcept -> bool { + return false; + } + + auto await_suspend(std::coroutine_handle<> coroutine) noexcept -> void { + m_schedule_operation.m_awaiting_coroutine_node.value() = coroutine; + m_schedule_operation.m_scheduler.m_scheduled_coroutines.push_back(m_schedule_operation.m_awaiting_coroutine_node); + m_schedule_operation.m_schedule_pos = std::prev(m_schedule_operation.m_scheduler.m_scheduled_coroutines.end()); + } + + auto await_resume() noexcept -> void { + m_schedule_operation.m_schedule_pos = std::monostate{}; + m_schedule_operation.m_scheduler.m_coro_instance_state.coroutine_stack_root.top_async_stack_frame = m_schedule_operation.m_async_stack_frame; + } + }; + + return awaiter{*this, m_scheduler.m_coro_instance_state.task_allocator}; } }; + return schedule_operation{*this}; } @@ -530,7 +548,7 @@ auto io_scheduler::schedule(duration_type timeout) noexcept -> kphp::coro::task< } // poll_op is not actually used here - kphp::coro::detail::poll_info poll_info{k2::INVALID_PLATFORM_DESCRIPTOR, kphp::coro::poll_op::read}; + kphp::coro::detail::poll_info poll_info{k2::INVALID_PLATFORM_DESCRIPTOR, kphp::coro::poll_op::read, m_coro_instance_state.task_allocator}; const auto cancellation_handler{make_cancellation_handler(poll_info)}; poll_info.m_schedule_position = add_timer_token(std::chrono::ceil(timeout), poll_info); co_await poll_info; @@ -556,7 +574,7 @@ auto io_scheduler::schedule(coroutine_type coroutine, duration_type timeout) noe } } - auto result{co_await kphp::coro::when_any(std::move(coroutine), make_timeout_task(std::chrono::ceil(timeout)))}; + auto result{co_await kphp::coro::when_any(std::move(coroutine), FORK_TASK(make_timeout_task(std::chrono::ceil(timeout))))}; if (std::holds_alternative(result)) [[unlikely]] { co_return std::unexpected{std::move(std::get<1>(result))}; } @@ -609,7 +627,7 @@ auto io_scheduler::poll(k2::descriptor descriptor, kphp::coro::poll_op poll_op, break; } - kphp::coro::detail::poll_info poll_info{descriptor, poll_op}; + kphp::coro::detail::poll_info poll_info{descriptor, poll_op, m_coro_instance_state.task_allocator}; const auto cancellation_handler{make_cancellation_handler(poll_info)}; if (timeout <= duration_type::zero()) { poll_info.m_schedule_position = m_parked_polls.emplace(poll_info.m_descriptor, poll_info); @@ -629,7 +647,7 @@ auto io_scheduler::accept(duration_type timeout) noexcept -> kphp::coro::task -struct promise_base : kphp::coro::async_stack_element { +struct promise_base : public kphp::coro::async_stack_element { private: struct not_started_tag {}; struct done_tag {}; @@ -61,7 +63,7 @@ struct promise_base : kphp::coro::async_stack_element { } auto& async_stack_root{*promise.get_async_stack_frame().async_stack_root}; - kphp::coro::resume(coroutine, async_stack_root); + kphp::coro::resume(coroutine, async_stack_root, promise.task_allocator()); } } @@ -104,9 +106,7 @@ struct promise_base : kphp::coro::async_stack_element { m_state = vk::intrusive::list>>{}; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto& async_stack_root{*get_async_stack_frame().async_stack_root}; - auto* prev_stack{m_task_allocator.exchange_stack(nullptr)}; - kphp::coro::resume(handle, async_stack_root); - m_task_allocator.set_stack(prev_stack); + kphp::coro::resume(handle, async_stack_root, m_task_allocator); } // coroutine already completed, don't suspend @@ -126,6 +126,10 @@ struct promise_base : kphp::coro::async_stack_element { return m_refcnt-- != 1; } + auto task_allocator() noexcept -> kphp::coro::detail::memory::task_allocator& { + return m_task_allocator; + } + template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { return kphp::coro::detail::memory::alloc(n); @@ -147,7 +151,7 @@ struct promise_base : kphp::coro::async_stack_element { }; template -class awaiter_base { +class awaiter_base : private kphp::coro::task_allocator_guard { void set_async_top_frame(async_stack_frame& caller_frame, void* return_address) noexcept { /** * shared_task is the top of the stack for calls from it. @@ -174,10 +178,12 @@ class awaiter_base { public: explicit awaiter_base(std::coroutine_handle coro) noexcept - : m_coro(coro) {} + : kphp::coro::task_allocator_guard(coro.promise().task_allocator()), + m_coro(coro) {} awaiter_base(awaiter_base&& other) noexcept - : m_awaiting_coroutine_node(std::move(other.m_awaiting_coroutine_node)), + : kphp::coro::task_allocator_guard(std::move(other)), + m_awaiting_coroutine_node(std::move(other.m_awaiting_coroutine_node)), m_coro(std::exchange(other.m_coro, {})) {} awaiter_base(const awaiter_base& other) = delete; @@ -313,11 +319,24 @@ struct shared_task final { } auto when_ready() const noexcept { - using awaiter_base = shared_task_impl::awaiter_base; - struct awaiter final : public awaiter_base { - using awaiter_base::awaiter_base; + struct awaitable { + private: + std::coroutine_handle m_coro; + + public: + explicit awaitable(std::coroutine_handle coro) noexcept + : m_coro{coro} {} + + auto operator co_await() noexcept { + using awaiter_base = shared_task_impl::awaiter_base; + struct awaiter final : public awaiter_base { + using awaiter_base::awaiter_base; + }; + return awaiter{m_coro}; + } }; - return awaiter{std::coroutine_handle::from_address(m_haddress)}; + + return awaitable{std::coroutine_handle::from_address(m_haddress)}; } auto get_handle() const noexcept -> std::coroutine_handle { @@ -357,7 +376,7 @@ struct shared_task final { } auto coro{std::coroutine_handle::from_address(m_haddress)}; if (!coro.promise().detach()) { - coro.destroy(); + kphp::coro::destroy(coro, coro.promise().task_allocator()); } } diff --git a/runtime-light/coroutine/task-allocator-guard.h b/runtime-light/coroutine/task-allocator-guard.h new file mode 100644 index 0000000000..0812e95ca1 --- /dev/null +++ b/runtime-light/coroutine/task-allocator-guard.h @@ -0,0 +1,31 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/segmented-stack-resource.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" + +namespace kphp::coro { + +class task_allocator_guard : private vk::not_copyable { +private: + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + memory_resource::segmented_stack_resource* m_stack{nullptr}; + +public: + task_allocator_guard() noexcept + : m_stack{m_task_allocator.exchange_stack(nullptr)} {} + + explicit task_allocator_guard(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : m_task_allocator{task_allocator}, + m_stack{m_task_allocator.exchange_stack(nullptr)} {} + + ~task_allocator_guard() { + m_task_allocator.set_stack(m_stack); + } +}; + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 9a8e8b62e9..0274b2c126 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -12,6 +12,7 @@ #include "common/containers/final_action.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/detail/allocator/task-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 55755b5025..722817e59b 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -150,7 +150,7 @@ class client final { while (!ctx.get()->interrupted.is_set()) { // Read response header or interrupt - auto read_header_res{co_await kphp::coro::when_any(t.get()->stream.read(resp_header_buf), interrupter)}; + auto read_header_res{co_await kphp::coro::when_any(FORK_TASK(t.get()->stream.read(resp_header_buf)), interrupter)}; // An interrupt has occurred if (std::holds_alternative(read_header_res)) [[unlikely]] { kphp::log::debug("reader has been interrupted"); diff --git a/runtime-light/streams/connection.h b/runtime-light/streams/connection.h index 4bd618b9bf..dbd725962a 100644 --- a/runtime-light/streams/connection.h +++ b/runtime-light/streams/connection.h @@ -140,7 +140,7 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s } const auto finalizer{vk::finally([state] noexcept { state.get()->m_unwatch_event.reset(); })}; - const auto v{co_await kphp::coro::when_any(unwatch_awaiter(std::move(state)), descriptor_awaiter(descriptor))}; + const auto v{co_await kphp::coro::when_any(FORK_TASK(unwatch_awaiter(std::move(state))), FORK_TASK(descriptor_awaiter(descriptor)))}; if (std::holds_alternative(v)) { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(h)); From ac39a52b502617f24d923c33272f3517dba32380 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 5 Oct 2026 15:45:47 +0300 Subject: [PATCH 06/26] imlement detached task --- compiler/code-gen/files/init-scripts.cpp | 2 +- compiler/code-gen/vertex-compiler.cpp | 2 +- .../confdata/state/instance-state.cpp | 6 +- .../components/kphp/state/instance-state.cpp | 4 +- runtime-light/coroutine/concepts.h | 12 +- runtime-light/coroutine/detached-task.h | 170 ++++++++++++++++++ .../detail/allocator/task-allocator.h | 17 -- .../coroutine/detail/type-traits-utils.h | 58 ++++++ runtime-light/coroutine/io-scheduler.h | 2 +- .../coroutine/task-allocator-guard.h | 11 +- runtime-light/coroutine/task.h | 42 +---- runtime-light/coroutine/type-traits.h | 39 +--- .../inter-component-session/client.h | 4 +- .../stdlib/curl/curl-easy-functions.h | 2 +- runtime-light/stdlib/file/resource.h | 4 +- .../stdlib/fork/wait-queue-functions.h | 4 +- runtime-light/stdlib/rpc/rpc-api.cpp | 2 +- .../stdlib/rpc/rpc-queue-functions.h | 4 +- .../stdlib/server/handler-functions.h | 2 +- runtime-light/stdlib/server/http-functions.h | 2 +- runtime-light/stdlib/time/timer-functions.h | 2 +- runtime-light/streams/connection.h | 5 +- 22 files changed, 281 insertions(+), 115 deletions(-) create mode 100644 runtime-light/coroutine/detached-task.h create mode 100644 runtime-light/coroutine/detail/type-traits-utils.h diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 619bd70360..8c9e1bbd1f 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -276,7 +276,7 @@ void InitScriptsCpp::compile(CodeGenerator& W) const { W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; if (G->is_output_mode_k2()) { - W << "run = FORK_TASK(" << FunctionName(main_file_id->main_function) << "$run());" << NL; + W << "run = DETACH_TASK(" << FunctionName(main_file_id->main_function) << "$run());" << NL; } else { W << "set_script (" << FunctionName(main_file_id->main_function) << "$run, " << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; } diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index 4e268fd83a..d04f9a760e 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -932,7 +932,7 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ if (mode == func_call_mode::fork_call) { if (func->is_interruptible) { - W << "(kphp::forks::start(FORK_TASK(" << FunctionName(func); + W << "(kphp::forks::start(DETACH_TASK(" << FunctionName(func); } else { W << FunctionForkName(func); } diff --git a/runtime-light/components/confdata/state/instance-state.cpp b/runtime-light/components/confdata/state/instance-state.cpp index 63bd32e98d..0551bf4757 100644 --- a/runtime-light/components/confdata/state/instance-state.cpp +++ b/runtime-light/components/confdata/state/instance-state.cpp @@ -230,7 +230,7 @@ auto InstanceState::erase_if_retired_and_unused(confdata_piece_list::iterator pi } auto InstanceState::init() noexcept -> void { - auto main_task{FORK_TASK(run())}; + auto main_task{DETACH_TASK(run())}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; main_task_async_stack_frame.async_stack_root = std::addressof(m_coroutine_instance_state.coroutine_stack_root); @@ -240,7 +240,7 @@ auto InstanceState::init() noexcept -> void { } auto InstanceState::run() noexcept -> kphp::coro::task<> { - co_await kphp::coro::when_all(FORK_TASK(service_loop()), FORK_TASK(accept_loop()), FORK_TASK(metrics_loop())); // all never return + co_await kphp::coro::when_all(DETACH_TASK(service_loop()), DETACH_TASK(accept_loop()), DETACH_TASK(metrics_loop())); // all never return kphp::log::assertion(false); } @@ -337,7 +337,7 @@ auto InstanceState::accept_loop() noexcept -> kphp::coro::task<> { } kphp::log::debug("accepted a stream: descriptor -> {}", stream->descriptor()); - if (!m_io_scheduler.start(FORK_TASK(serve_reader_lease(std::move(*stream))))) [[unlikely]] { + if (!m_io_scheduler.start(DETACH_TASK(serve_reader_lease(std::move(*stream))))) [[unlikely]] { kphp::log::warning("failed to serve a confdata reader lease"); } } diff --git a/runtime-light/components/kphp/state/instance-state.cpp b/runtime-light/components/kphp/state/instance-state.cpp index 8d8bda794a..2a26f9c1ce 100644 --- a/runtime-light/components/kphp/state/instance-state.cpp +++ b/runtime-light/components/kphp/state/instance-state.cpp @@ -62,10 +62,10 @@ void InstanceState::init_script_execution() noexcept { kphp::coro::task<> script_task; init_php_scripts_in_each_worker(php_script_mutable_globals_singleton, script_task); - auto main_task{FORK_TASK(std::invoke( + auto main_task{DETACH_TASK(std::invoke( [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { // wrap script with additional check for unhandled exception - script_task = FORK_TASK(std::invoke( + script_task = DETACH_TASK(std::invoke( [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { co_await script_task; if (auto exception{std::move(ForkInstanceState::get().current_info().get().thrown_exception)}; !exception.is_null()) [[unlikely]] { diff --git a/runtime-light/coroutine/concepts.h b/runtime-light/coroutine/concepts.h index aa3b507b24..d5c6a3847c 100644 --- a/runtime-light/coroutine/concepts.h +++ b/runtime-light/coroutine/concepts.h @@ -9,6 +9,8 @@ #include #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/detail/type-traits-utils.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" @@ -44,6 +46,9 @@ concept global_co_await_awaitable = requires(T&& t) { template concept awaitable = detail::member_co_await_awaitable || detail::global_co_await_awaitable || detail::awaiter; +template +concept detached_awaitable = awaitable && !is_task_v; + namespace detail { template @@ -60,10 +65,13 @@ auto get_awaiter(awaitable_type&& value) noexcept { } // namespace detail template -concept coroutine = awaitable && (requires { - { static_cast>(std::declval()) }; +concept coroutine = awaitable && (is_task_v || requires { + { static_cast>(std::declval()) }; } || requires { { static_cast>(std::declval()) }; }); +template +concept detached_coroutine = coroutine && !is_task_v; + } // namespace kphp::coro::concepts diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h new file mode 100644 index 0000000000..dad79eabdf --- /dev/null +++ b/runtime-light/coroutine/detached-task.h @@ -0,0 +1,170 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +#include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" +#include "runtime-light/coroutine/task.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::coro { + +namespace detail { + +template +class awaiter_base { + void push_async_stack_frame(async_stack_frame& caller_frame, void* return_address) noexcept { + async_stack_frame& callee_frame{m_coro.promise().get_async_stack_frame()}; + callee_frame.caller_async_stack_frame = std::addressof(caller_frame); + callee_frame.return_address = return_address; + + auto* async_stack_root{caller_frame.async_stack_root}; + kphp::log::assertion(async_stack_root != nullptr); + callee_frame.async_stack_root = async_stack_root; + async_stack_root->top_async_stack_frame = std::addressof(callee_frame); + } + + void detach_from_async_stack() noexcept { + async_stack_frame& callee_frame{m_coro.promise().get_async_stack_frame()}; + callee_frame.caller_async_stack_frame = nullptr; + } + +protected: + bool m_started{}; + bool m_suspended{}; + std::coroutine_handle m_coro{}; + +public: + explicit awaiter_base(std::coroutine_handle coro) noexcept + : m_coro(coro) {} + + awaiter_base(awaiter_base&& other) noexcept + : m_started(other.m_started), + m_suspended(std::exchange(other.m_suspended, false)), + m_coro(std::exchange(other.m_coro, {})) {} + + awaiter_base(const awaiter_base& other) = delete; + awaiter_base& operator=(const awaiter_base& other) = delete; + awaiter_base& operator=(awaiter_base&& other) = delete; + + ~awaiter_base() { + if (m_coro != nullptr && m_suspended) { + m_coro.promise().m_next = nullptr; + detach_from_async_stack(); + } + } + + auto await_ready() noexcept -> bool { + kphp::log::assertion(!std::exchange(m_started, true)); // to make sure it's not co_awaited more than once + return false; + } + + template caller_promise_type> + [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> std::coroutine_handle { + push_async_stack_frame(awaiting_coroutine.promise().get_async_stack_frame(), STACK_RETURN_ADDRESS); + m_coro.promise().m_next = awaiting_coroutine.address(); + m_suspended = true; + return m_coro; + } + + auto await_resume() noexcept -> void { + m_suspended = false; + } +}; + +template +auto make_fork_task(kphp::coro::task&& task) noexcept { + return kphp::coro::detached_task{std::move(task)}; +} + +} // namespace detail + +template +struct detached_task { +private: + using task_promise_type = kphp::coro::task::promise_type; + + explicit detached_task(kphp::coro::task&& task) noexcept + : m_coro(std::exchange(task.m_coro, {})) {} + +public: + detached_task() noexcept = default; + + detached_task(const detached_task& other) noexcept = delete; + + detached_task(detached_task&& other) noexcept + : m_coro(std::exchange(other.m_coro, {})), + m_stack(std::exchange(other.m_stack, nullptr)) {} + + detached_task& operator=(const detached_task& other) noexcept = delete; + + detached_task& operator=(detached_task&& other) noexcept { + std::swap(m_coro, other.m_coro); + std::swap(m_stack, other.m_stack); + + return *this; + } + + ~detached_task() { + if (m_coro) { + auto* prev_stack{m_task_allocator.exchange_stack(m_stack)}; + m_coro.destroy(); + m_task_allocator.set_stack(prev_stack); + } + } + + constexpr auto operator co_await() noexcept { + using awaiter_base = detail::awaiter_base; + struct awaiter final : public awaiter_base, private kphp::coro::task_allocator_guard { + awaiter(std::coroutine_handle coro, kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::segmented_stack_resource* stack) noexcept + : awaiter_base(coro), + kphp::coro::task_allocator_guard(task_allocator, stack) {} + + auto await_resume() noexcept -> T { + awaiter_base::await_resume(); + return awaiter_base::m_coro.promise().result(); + } + }; + + return awaiter{m_coro, m_task_allocator, m_stack}; + } + + auto get_handle() noexcept -> std::coroutine_handle { + return m_coro; + } + + // conversion functions + // + // erase type + explicit operator detached_task<>() && noexcept { + return detached_task<>{std::exchange(m_coro, {})}; + } + + // restore erased type + template + requires(std::same_as) + explicit operator detached_task() && noexcept { + return detached_task{std::exchange(m_coro, {})}; + } + +private: + std::coroutine_handle m_coro; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + memory_resource::segmented_stack_resource* m_stack{nullptr}; + + template + friend auto detail::make_fork_task(kphp::coro::task&& task) noexcept; +}; + +} // namespace kphp::coro + +#define DETACH_TASK(...) (kphp::coro::task_allocator_guard{}, kphp::coro::detail::make_fork_task(__VA_ARGS__)) diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h index 63d936613e..4b6474ee72 100644 --- a/runtime-light/coroutine/detail/allocator/task-allocator.h +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -124,23 +124,6 @@ struct task_allocator final : private vk::not_copyable { return prev; } - auto fork() noexcept { - struct fork_guard { - kphp::coro::detail::memory::task_allocator& m_task_allocator; - memory_resource::segmented_stack_resource* m_prev_stack{nullptr}; - - explicit fork_guard(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept - : m_task_allocator{task_allocator}, - m_prev_stack{task_allocator.exchange_stack(nullptr)} {} - - ~fork_guard() { - m_task_allocator.set_stack(m_prev_stack); - } - }; - - return fork_guard{*this}; - } - auto alloc_script_memory(size_t size) noexcept -> void* { kphp::log::assertion(size != 0); kphp::log::assertion(m_curr_stack != nullptr); diff --git a/runtime-light/coroutine/detail/type-traits-utils.h b/runtime-light/coroutine/detail/type-traits-utils.h new file mode 100644 index 0000000000..5adaf3fc9b --- /dev/null +++ b/runtime-light/coroutine/detail/type-traits-utils.h @@ -0,0 +1,58 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/shared-task.h" +#include "runtime-light/coroutine/task.h" + +namespace kphp::coro { + +namespace detail { + +template +requires std::invocable +class async_function_return_type { + using return_type = std::invoke_result_t; + + template + struct task_inner { + using type = U; + }; + + template + struct task_inner> { + using type = U; + }; + + template + struct task_inner> { + using type = U; + }; + + template + struct task_inner> { + using type = U; + }; + +public: + using type = task_inner::type; +}; + +template +struct is_task : std::false_type {}; + +template +struct is_task> : std::true_type {}; + +} // namespace detail + +template +inline constexpr bool is_task_v = kphp::coro::detail::is_task::value; + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index bf9d084789..58e545c505 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -574,7 +574,7 @@ auto io_scheduler::schedule(coroutine_type coroutine, duration_type timeout) noe } } - auto result{co_await kphp::coro::when_any(std::move(coroutine), FORK_TASK(make_timeout_task(std::chrono::ceil(timeout))))}; + auto result{co_await kphp::coro::when_any(std::move(coroutine), DETACH_TASK(make_timeout_task(std::chrono::ceil(timeout))))}; if (std::holds_alternative(result)) [[unlikely]] { co_return std::unexpected{std::move(std::get<1>(result))}; } diff --git a/runtime-light/coroutine/task-allocator-guard.h b/runtime-light/coroutine/task-allocator-guard.h index 0812e95ca1..d770ee0470 100644 --- a/runtime-light/coroutine/task-allocator-guard.h +++ b/runtime-light/coroutine/task-allocator-guard.h @@ -16,12 +16,15 @@ class task_allocator_guard : private vk::not_copyable { memory_resource::segmented_stack_resource* m_stack{nullptr}; public: - task_allocator_guard() noexcept - : m_stack{m_task_allocator.exchange_stack(nullptr)} {} + explicit task_allocator_guard( + memory_resource::segmented_stack_resource* new_stack = nullptr) noexcept + : m_stack{m_task_allocator.exchange_stack(new_stack)} {} - explicit task_allocator_guard(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + explicit task_allocator_guard( + kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::segmented_stack_resource* new_stack = nullptr) noexcept : m_task_allocator{task_allocator}, - m_stack{m_task_allocator.exchange_stack(nullptr)} {} + m_stack{m_task_allocator.exchange_stack(new_stack)} {} ~task_allocator_guard() { m_task_allocator.set_stack(m_stack); diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 0274b2c126..f56485c2d9 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -12,13 +12,14 @@ #include "common/containers/final_action.h" #include "runtime-light/coroutine/async-stack.h" -#include "runtime-light/coroutine/control-functions.h" -#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/detail/allocator/task-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { +template +struct detached_task; + namespace task_impl { template @@ -76,8 +77,6 @@ struct promise_base : kphp::coro::async_stack_element { } void* m_next{}; - memory_resource::segmented_stack_resource* m_stack{}; - kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; }; template @@ -102,7 +101,6 @@ class awaiter_base { bool m_started{}; bool m_suspended{}; std::coroutine_handle m_coro{}; - memory_resource::segmented_stack_resource* m_prev_stack{}; public: explicit awaiter_base(std::coroutine_handle coro) noexcept @@ -131,7 +129,6 @@ class awaiter_base { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> std::coroutine_handle { - m_prev_stack = m_coro.promise().m_task_allocator.exchange_stack(m_coro.promise().m_stack); push_async_stack_frame(awaiting_coroutine.promise().get_async_stack_frame(), STACK_RETURN_ADDRESS); m_coro.promise().m_next = awaiting_coroutine.address(); m_suspended = true; @@ -139,7 +136,6 @@ class awaiter_base { } auto await_resume() noexcept -> void { - m_coro.promise().m_task_allocator.set_stack(m_prev_stack); m_suspended = false; } }; @@ -165,29 +161,18 @@ struct task { : m_coro(coro) {} task(const task& other) noexcept = delete; - - task(task&& other) noexcept - : m_coro(std::exchange(other.m_coro, {})) {} - + task(task&& other) noexcept = delete; task& operator=(const task& other) noexcept = delete; - - task& operator=(task&& other) noexcept { - std::swap(m_coro, other.m_coro); - return *this; - } + task& operator=(task&& other) noexcept = delete; ~task() { if (m_coro) { - auto& task_allocator{m_coro.promise().m_task_allocator}; - auto* prev_stack{task_allocator.exchange_stack(m_coro.promise().m_stack)}; m_coro.destroy(); - task_allocator.set_stack(prev_stack); } } struct promise_base : task_impl::promise_base { auto get_return_object() noexcept -> task { - this->m_stack = this->m_task_allocator.current_stack(); return task{std::coroutine_handle::from_promise(*static_cast(this))}; } @@ -235,23 +220,10 @@ struct task { return m_coro; } - // conversion functions - // - // erase type - explicit operator task<>() && noexcept { - return task<>{std::exchange(m_coro, {})}; - } - // restore erased type - template - requires(std::same_as) - explicit operator task() && noexcept { - return task{std::exchange(m_coro, {})}; - } - private: std::coroutine_handle m_coro{}; + + friend struct kphp::coro::detached_task; }; } // namespace kphp::coro - -#define FORK_TASK(...) (kphp::coro::detail::memory::task_allocator::get().fork(), __VA_ARGS__) diff --git a/runtime-light/coroutine/type-traits.h b/runtime-light/coroutine/type-traits.h index ad2bf0569f..12393178af 100644 --- a/runtime-light/coroutine/type-traits.h +++ b/runtime-light/coroutine/type-traits.h @@ -9,44 +9,13 @@ #include #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/detail/type-traits-utils.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" namespace kphp::coro { -namespace type_traits_impl { - -template -requires std::invocable -class async_function_return_type { - using return_type = std::invoke_result_t; - - template - struct task_inner { - using type = U; - }; - - template - struct task_inner> { - using type = U; - }; - - template - struct task_inner> { - using type = U; - }; - -public: - using type = task_inner::type; -}; - -} // namespace type_traits_impl - -template -inline constexpr bool is_task_v = requires { - { static_cast>(std::declval()) }; -}; - template inline constexpr bool is_task_function_v = requires { requires std::invocable && is_task_v>; }; @@ -54,13 +23,15 @@ template inline constexpr bool is_async_function_v = is_task_function_v || requires { requires std::invocable; { static_cast>(std::declval>()) }; +} || requires { + { static_cast>(std::declval>()) }; }; // ================================================================================================ template requires std::invocable -using async_function_return_type_t = kphp::coro::type_traits_impl::async_function_return_type::type; +using async_function_return_type_t = kphp::coro::detail::async_function_return_type::type; // ================================================================================================ diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 722817e59b..8ba88abc71 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -110,7 +110,7 @@ class client final { // The protocol design assumes that interrupting the transfer in the middle of a frame leads to critical error. // Therefore, we need to write the request in a separate coroutine. // This technique prevents integrity violations when this coroutine is cancelled. - kphp::coro::io_scheduler::get().start(FORK_TASK(process_write(t, qid, payload))); + kphp::coro::io_scheduler::get().start(DETACH_TASK(process_write(t, qid, payload))); co_await req_finish_notifier[qid]; kphp::log::assertion(req_status.contains(qid)); @@ -150,7 +150,7 @@ class client final { while (!ctx.get()->interrupted.is_set()) { // Read response header or interrupt - auto read_header_res{co_await kphp::coro::when_any(FORK_TASK(t.get()->stream.read(resp_header_buf)), interrupter)}; + auto read_header_res{co_await kphp::coro::when_any(DETACH_TASK(t.get()->stream.read(resp_header_buf)), interrupter)}; // An interrupt has occurred if (std::holds_alternative(read_header_res)) [[unlikely]] { kphp::log::debug("reader has been interrupted"); diff --git a/runtime-light/stdlib/curl/curl-easy-functions.h b/runtime-light/stdlib/curl/curl-easy-functions.h index 81ae6e7912..4652fe8b07 100644 --- a/runtime-light/stdlib/curl/curl-easy-functions.h +++ b/runtime-light/stdlib/curl/curl-easy-functions.h @@ -475,7 +475,7 @@ inline auto f$curl_exec_concurrently(kphp::web::curl::easy_type easy_id, double auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; auto sched_res{co_await kphp::coro::io_scheduler::get().schedule( - FORK_TASK(kphp::forks::id_managed(kphp::web::simple::perform, kphp::web::simple::transfer{easy_id})), timeout)}; + DETACH_TASK(kphp::forks::id_managed(kphp::web::simple::perform, kphp::web::simple::transfer{easy_id})), timeout)}; if (!sched_res.has_value()) [[unlikely]] { kphp::web::curl::print_debug( "could not execute curl easy handle concurrently", diff --git a/runtime-light/stdlib/file/resource.h b/runtime-light/stdlib/file/resource.h index 04e5fe8b87..cc2f549e92 100644 --- a/runtime-light/stdlib/file/resource.h +++ b/runtime-light/stdlib/file/resource.h @@ -637,7 +637,7 @@ class socket : public async_resource { socket& operator=(socket&& other) noexcept { if (this != std::addressof(other)) { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(FORK_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); + kphp::coro::io_scheduler::get().spawn(DETACH_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); } m_open = std::exchange(other.m_open, false); @@ -649,7 +649,7 @@ class socket : public async_resource { ~socket() override { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(FORK_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); + kphp::coro::io_scheduler::get().spawn(DETACH_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); } } diff --git a/runtime-light/stdlib/fork/wait-queue-functions.h b/runtime-light/stdlib/fork/wait-queue-functions.h index 1ffb860803..4c56bf54b9 100644 --- a/runtime-light/stdlib/fork/wait-queue-functions.h +++ b/runtime-light/stdlib/fork/wait-queue-functions.h @@ -56,7 +56,7 @@ inline kphp::coro::task> wait_queue_next(int64_t queue_id timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(FORK_TASK(wait_queue_next_task(await_set.next())), timeout)}; + auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(DETACH_TASK(wait_queue_next_task(await_set.next())), timeout)}; if (!wait_result) { co_return std::nullopt; } @@ -94,7 +94,7 @@ inline void wait_queue_push(int64_t queue_id, int64_t fork_id) noexcept { }}; auto& await_set{(*opt_await_set).get()}; - await_set.push(FORK_TASK(wait_queue_wrapper_task(std::move(fork_task), fork_id))); + await_set.push(DETACH_TASK(wait_queue_wrapper_task(std::move(fork_task), fork_id))); } } // namespace kphp::forks diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp index 6a6d95dd01..012dd5c867 100644 --- a/runtime-light/stdlib/rpc/rpc-api.cpp +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -358,7 +358,7 @@ kphp::rpc::query_info send_request(std::string_view actor, std::optional return {reinterpret_cast(response_exp->buffer()), size}; }}; - auto fetch_task{FORK_TASK(kphp::rpc::query::response(std::move(q), response_buffer_provider))}; + auto fetch_task{DETACH_TASK(kphp::rpc::query::response(std::move(q), response_buffer_provider))}; auto fetch_result{co_await kphp::coro::io_scheduler::get().schedule(std::move(fetch_task))}; if (!fetch_result) [[unlikely]] { response_exp = std::unexpected{fetch_result.error()}; diff --git a/runtime-light/stdlib/rpc/rpc-queue-functions.h b/runtime-light/stdlib/rpc/rpc-queue-functions.h index 27eb74307d..0daa7bb0f9 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-functions.h +++ b/runtime-light/stdlib/rpc/rpc-queue-functions.h @@ -43,7 +43,7 @@ inline void rpc_queue_push(int64_t queue_id, int64_t request_id) noexcept { } auto& await_set{(*opt_await_set).get()}; - await_set.push(FORK_TASK(rpc_queue_wrapper_task(static_cast>(it_awaiter_task->second), request_id))); + await_set.push(DETACH_TASK(rpc_queue_wrapper_task(static_cast>(it_awaiter_task->second), request_id))); } inline int64_t rpc_queue_create(std::span request_ids) noexcept { @@ -83,7 +83,7 @@ inline kphp::coro::task> rpc_queue_next(int64_t queue_id, timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(FORK_TASK(rpc_queue_next_task(await_set.next())), timeout)}; + const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(DETACH_TASK(rpc_queue_next_task(await_set.next())), timeout)}; if (!expected_next) { co_return std::nullopt; } diff --git a/runtime-light/stdlib/server/handler-functions.h b/runtime-light/stdlib/server/handler-functions.h index b5a97d14d8..452bac03ab 100644 --- a/runtime-light/stdlib/server/handler-functions.h +++ b/runtime-light/stdlib/server/handler-functions.h @@ -19,7 +19,7 @@ void f$register_shutdown_function(F&& f, Args&&... args) noexcept { // it's a lambda coroutine, so: // 1. don't capture anything; // 2. parameters are passed by value. - auto shutdown_function_task{FORK_TASK(std::invoke( + auto shutdown_function_task{DETACH_TASK(std::invoke( [](F f, Args... args) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f), std::move(args)...); diff --git a/runtime-light/stdlib/server/http-functions.h b/runtime-light/stdlib/server/http-functions.h index a6f086bdc0..e50c807603 100644 --- a/runtime-light/stdlib/server/http-functions.h +++ b/runtime-light/stdlib/server/http-functions.h @@ -75,7 +75,7 @@ bool f$header_register_callback(F&& f) noexcept { return false; } - auto headers_callback_task{FORK_TASK(std::invoke( + auto headers_callback_task{DETACH_TASK(std::invoke( [](F f) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f)); diff --git a/runtime-light/stdlib/time/timer-functions.h b/runtime-light/stdlib/time/timer-functions.h index 00259c6b74..e399df98c6 100644 --- a/runtime-light/stdlib/time/timer-functions.h +++ b/runtime-light/stdlib/time/timer-functions.h @@ -24,7 +24,7 @@ void f$set_timer(int64_t timeout_ms, T&& on_timer_callback) noexcept { // TODO choose from: // 1. someone should pop that fork from ForkInstanceState since it will stay there unless we perform f$wait on fork // 2. start timer_task via kphp::coro::io_scheduler::spawn (it won't have distinct fork id) - auto timer_task{FORK_TASK(std::invoke( + auto timer_task{DETACH_TASK(std::invoke( [](std::chrono::milliseconds duration, T on_timer_callback) noexcept -> kphp::coro::task<> { co_await kphp::forks::id_managed([](std::chrono::milliseconds duration) noexcept { return kphp::coro::io_scheduler::get().schedule(duration); }, duration); diff --git a/runtime-light/streams/connection.h b/runtime-light/streams/connection.h index dbd725962a..42762d298e 100644 --- a/runtime-light/streams/connection.h +++ b/runtime-light/streams/connection.h @@ -140,7 +140,7 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s } const auto finalizer{vk::finally([state] noexcept { state.get()->m_unwatch_event.reset(); })}; - const auto v{co_await kphp::coro::when_any(FORK_TASK(unwatch_awaiter(std::move(state))), FORK_TASK(descriptor_awaiter(descriptor)))}; + const auto v{co_await kphp::coro::when_any(DETACH_TASK(unwatch_awaiter(std::move(state))), DETACH_TASK(descriptor_awaiter(descriptor)))}; if (std::holds_alternative(v)) { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(h)); @@ -151,7 +151,8 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s }}; m_shared_state.get()->m_unwatch_event.emplace(); - if (!kphp::coro::io_scheduler::get().spawn(FORK_TASK(watcher(m_stream.descriptor(), m_shared_state, std::forward(h))))) [[unlikely]] { + if (!kphp::coro::io_scheduler::get().spawn(DETACH_TASK(watcher(m_stream.descriptor(), m_shared_state, std::forward(h))))) + [[unlikely]] { m_shared_state.get()->m_unwatch_event.reset(); return std::unexpected{k2::errno_ebusy}; } From d3a18c9a1db14d9a04f758a7cc4cfdaf7b1c443c Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 5 Oct 2026 15:50:55 +0300 Subject: [PATCH 07/26] add constraints for task type --- runtime-light/coroutine/await-set.h | 3 ++- runtime-light/coroutine/detail/await-set.h | 3 ++- .../coroutine/detail/task-self-deleting.h | 2 +- runtime-light/coroutine/detail/when-all.h | 2 +- runtime-light/coroutine/detail/when-any.h | 2 +- runtime-light/coroutine/io-scheduler.h | 16 ++++++++-------- runtime-light/coroutine/when-all.h | 2 +- runtime-light/coroutine/when-any.h | 2 +- runtime-light/stdlib/fork/fork-functions.h | 2 +- 9 files changed, 18 insertions(+), 16 deletions(-) diff --git a/runtime-light/coroutine/await-set.h b/runtime-light/coroutine/await-set.h index 3117d46cc7..1f4592bb94 100644 --- a/runtime-light/coroutine/await-set.h +++ b/runtime-light/coroutine/await-set.h @@ -46,7 +46,8 @@ class await_set { await_set& operator=(const await_set&) = delete; template - requires kphp::coro::concepts::awaitable && std::is_same_v::awaiter_return_type, return_type> + requires kphp::coro::concepts::detached_awaitable && + std::is_same_v::awaiter_return_type, return_type> void push(awaitable_type awaitable) noexcept { kphp::log::assertion(m_await_broker != nullptr); m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable)), m_coroutine_stack_root, STACK_RETURN_ADDRESS); diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index bf0bf60d5a..c5f315afbf 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -13,6 +13,7 @@ #include "common/containers/intrusive-list.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" @@ -386,7 +387,7 @@ class await_set_awaitable { ~await_set_awaitable() = default; }; -template +template auto make_await_set_task(awaitable_type coroutine) noexcept -> await_set_task::awaiter_return_type> { if constexpr (std::is_void_v::awaiter_return_type>) { co_await std::move(coroutine); diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index ec8ddea722..1d0bd148ea 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -107,7 +107,7 @@ inline auto promise_self_deleting::get_return_object_on_allocation_failure() noe } // namespace task_self_deleting -template +template auto make_task_self_deleting(awaitable_type awaitable) noexcept -> task_self_deleting::task_self_deleting { co_await std::move(awaitable); co_return; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index 4af87fdd20..c214e0cfe1 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -310,7 +310,7 @@ class when_all_task { } }; -template +template auto make_when_all_task(awaitable_type awaitable) noexcept -> when_all_task::awaiter_return_type> { if constexpr (std::is_void_v::awaiter_return_type>) { co_await std::move(awaitable); diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 75807a3fea..00be6508a4 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -330,7 +330,7 @@ class when_any_task { } }; -template +template auto make_when_any_task(awaitable_type awaitable) noexcept -> when_any_task::awaiter_return_type> { if constexpr (std::is_void_v::awaiter_return_type>) { co_await std::move(awaitable); diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 58e545c505..9c326513d1 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -99,7 +99,7 @@ class io_scheduler { * @param coroutine The coroutine to spawn. * @return True if the coroutine was successfully spawned, false otherwise. */ - template + template auto spawn(coroutine_type coroutine) noexcept -> bool; /** @@ -107,7 +107,7 @@ class io_scheduler { * @param corotuine The coroutine to start. * @return True if the coroutine was successfully started, false otherwise. */ - template + template auto start(coroutine_type coroutine) noexcept -> bool; /** @@ -129,7 +129,7 @@ class io_scheduler { * @param coroutine The coroutine to schedule. * @return A task that will yield the coroutine's return value when completed. */ - template + template [[nodiscard]] auto schedule(coroutine_type coroutine) noexcept -> kphp::coro::task::return_type>; /** * @brief Schedules a coroutine with a timeout. @@ -139,7 +139,7 @@ class io_scheduler { * - Positive: rounded up to the next whole millisecond. * @return A task that yields either the coroutine's result or kphp::coro::timeout_status::timeout. */ - template + template [[nodiscard]] auto schedule(coroutine_type coroutine, duration_type timeout) noexcept -> kphp::coro::task::return_type, timeout_status>>; @@ -446,7 +446,7 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { return empty() ? k2::PollStatus::PollFinishedOk : k2::PollStatus::PollReschedule; } -template +template auto io_scheduler::spawn(coroutine_type coroutine) noexcept -> bool { auto owned_task{kphp::coro::detail::make_task_self_deleting(std::move(coroutine))}; auto& coroutine_node{owned_task.get_coroutine_node()}; @@ -457,7 +457,7 @@ auto io_scheduler::spawn(coroutine_type coroutine) noexcept -> bool { return true; } -template +template auto io_scheduler::start(coroutine_type coroutine) noexcept -> bool { auto owned_task{kphp::coro::detail::make_task_self_deleting(std::move(coroutine))}; auto handle{owned_task.get_handle()}; @@ -554,13 +554,13 @@ auto io_scheduler::schedule(duration_type timeout) noexcept -> kphp::coro::task< co_await poll_info; } -template +template auto io_scheduler::schedule(coroutine_type coroutine) noexcept -> kphp::coro::task::return_type> { co_await schedule(); co_return co_await std::move(coroutine); } -template +template auto io_scheduler::schedule(coroutine_type coroutine, duration_type timeout) noexcept -> kphp::coro::task::return_type, timeout_status>> { using expected_return_type = typename kphp::coro::coroutine_traits::return_type; diff --git a/runtime-light/coroutine/when-all.h b/runtime-light/coroutine/when-all.h index 0803c8758d..cc98439a31 100644 --- a/runtime-light/coroutine/when-all.h +++ b/runtime-light/coroutine/when-all.h @@ -13,7 +13,7 @@ namespace kphp::coro { -template +template [[nodiscard]] auto when_all(awaitable_types... awaitables) noexcept { return detail::when_all::when_all_ready_awaitable< std::tuple::awaiter_return_type>...>>{ diff --git a/runtime-light/coroutine/when-any.h b/runtime-light/coroutine/when-any.h index 2136436115..a38c70ea02 100644 --- a/runtime-light/coroutine/when-any.h +++ b/runtime-light/coroutine/when-any.h @@ -13,7 +13,7 @@ namespace kphp::coro { -template +template [[nodiscard]] auto when_any(awaitable_types... awaitables) noexcept { return detail::when_any::when_any_ready_awaitable< std::tuple::awaiter_return_type>...>>{ diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index 2ed12e1888..be098a5ef5 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -44,7 +44,7 @@ class scoped_id_managed { scoped_id_managed& operator=(scoped_id_managed&&) = delete; }; -template +template auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task::awaiter_return_type> { auto& fork_instance_st{ForkInstanceState::get()}; const auto saved_fork_id{fork_instance_st.current_id}; From e91508bd5b024aef6e75e6d84d94dc5a665668ac Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 5 Oct 2026 17:38:26 +0300 Subject: [PATCH 08/26] move code gen and runtime to new api --- compiler/code-gen/files/init-scripts.cpp | 4 +-- compiler/code-gen/vertex-compiler.cpp | 15 ++++++++--- .../confdata/state/instance-state.cpp | 8 +++--- .../components/kphp/state/instance-state.cpp | 14 +++++------ .../components/kphp/state/instance-state.h | 2 +- runtime-light/core/globals/php-init-scripts.h | 4 +-- runtime-light/coroutine/detached-task.h | 25 +++++++++++-------- runtime-light/coroutine/io-scheduler.h | 3 ++- runtime-light/server/http/http-server-state.h | 2 +- .../inter-component-session/client.h | 5 ++-- .../stdlib/curl/curl-easy-functions.h | 5 +++- runtime-light/stdlib/file/resource.h | 5 ++-- runtime-light/stdlib/fork/fork-functions.h | 3 ++- runtime-light/stdlib/fork/fork-state.h | 6 ++--- .../stdlib/fork/wait-queue-functions.h | 5 ++-- runtime-light/stdlib/rpc/rpc-api.cpp | 3 ++- .../stdlib/rpc/rpc-queue-functions.h | 4 +-- .../stdlib/server/handler-functions.h | 4 +-- runtime-light/stdlib/server/http-functions.h | 4 +-- runtime-light/stdlib/time/timer-functions.h | 4 +-- runtime-light/streams/connection.h | 6 +++-- 21 files changed, 79 insertions(+), 52 deletions(-) diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 8c9e1bbd1f..60e0ee3255 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -259,7 +259,7 @@ void InitScriptsCpp::compile(CodeGenerator& W) const { if (G->is_output_mode_k2()) { FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() - << ", kphp::coro::task<> &run" + << ", kphp::coro::detached_task<> &run" ")" << BEGIN; } else { @@ -276,7 +276,7 @@ void InitScriptsCpp::compile(CodeGenerator& W) const { W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; if (G->is_output_mode_k2()) { - W << "run = DETACH_TASK(" << FunctionName(main_file_id->main_function) << "$run());" << NL; + W << "run = kphp::coro::detach_task(" << FunctionName(main_file_id->main_function) << "$run);" << NL; } else { W << "set_script (" << FunctionName(main_file_id->main_function) << "$run, " << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; } diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index d04f9a760e..93583db2c9 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -932,7 +932,7 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ if (mode == func_call_mode::fork_call) { if (func->is_interruptible) { - W << "(kphp::forks::start(DETACH_TASK(" << FunctionName(func); + W << "(kphp::forks::start(kphp::coro::detach_task([](auto&&... args) noexcept { return " << FunctionName(func); } else { W << FunctionForkName(func); } @@ -947,7 +947,12 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ const TypeData* tp = tinf::get_type(root); W << "< " << TypeName(tp) << " >"; } - W << "("; + + if (func->is_interruptible && mode == func_call_mode::fork_call) { + W << "(std::forward(args)...); }"; + } else { + W << "("; + } if (func && func->is_extern() && vk::any_of_equal(func->name, "JsonEncoder$$to_json_impl", "JsonEncoder$$from_json_impl")) { root = patch_compiling_json_impl_call(W, root); @@ -962,6 +967,10 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ } } + if (!args.empty() && func->is_interruptible && mode == func_call_mode::fork_call) { + W << ", "; + } + W << JoinValues(args, ", "); if (is_function_call_should_be_tracked(func)) { W << "))"; @@ -969,7 +978,7 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ W << ")"; if (func->is_interruptible) { if (mode == func_call_mode::fork_call) { - W << ")))"; + W << "))"; } else { W << ")"; } diff --git a/runtime-light/components/confdata/state/instance-state.cpp b/runtime-light/components/confdata/state/instance-state.cpp index 0551bf4757..92b9488271 100644 --- a/runtime-light/components/confdata/state/instance-state.cpp +++ b/runtime-light/components/confdata/state/instance-state.cpp @@ -25,6 +25,7 @@ #include "runtime-light/components/confdata/confdata-proxy/sync-functions.h" #include "runtime-light/components/confdata/confdata-proxy/tl.h" #include "runtime-light/components/confdata/state/component-state.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/coroutine/when-all.h" @@ -230,7 +231,7 @@ auto InstanceState::erase_if_retired_and_unused(confdata_piece_list::iterator pi } auto InstanceState::init() noexcept -> void { - auto main_task{DETACH_TASK(run())}; + auto main_task{kphp::coro::detach_task(&InstanceState::run, this)}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; main_task_async_stack_frame.async_stack_root = std::addressof(m_coroutine_instance_state.coroutine_stack_root); @@ -240,7 +241,8 @@ auto InstanceState::init() noexcept -> void { } auto InstanceState::run() noexcept -> kphp::coro::task<> { - co_await kphp::coro::when_all(DETACH_TASK(service_loop()), DETACH_TASK(accept_loop()), DETACH_TASK(metrics_loop())); // all never return + co_await kphp::coro::when_all(kphp::coro::detach_task(&InstanceState::service_loop, this), kphp::coro::detach_task(&InstanceState::accept_loop, this), + kphp::coro::detach_task(&InstanceState::metrics_loop, this)); // all never return kphp::log::assertion(false); } @@ -337,7 +339,7 @@ auto InstanceState::accept_loop() noexcept -> kphp::coro::task<> { } kphp::log::debug("accepted a stream: descriptor -> {}", stream->descriptor()); - if (!m_io_scheduler.start(DETACH_TASK(serve_reader_lease(std::move(*stream))))) [[unlikely]] { + if (!m_io_scheduler.start(kphp::coro::detach_task(&InstanceState::serve_reader_lease, this, std::move(*stream)))) [[unlikely]] { kphp::log::warning("failed to serve a confdata reader lease"); } } diff --git a/runtime-light/components/kphp/state/instance-state.cpp b/runtime-light/components/kphp/state/instance-state.cpp index 2a26f9c1ce..6eebf8e615 100644 --- a/runtime-light/components/kphp/state/instance-state.cpp +++ b/runtime-light/components/kphp/state/instance-state.cpp @@ -59,23 +59,23 @@ consteval std::string_view resolve_sapi_name() noexcept { void InstanceState::init_script_execution() noexcept { runtime_context.init(); - kphp::coro::task<> script_task; + kphp::coro::detached_task<> script_task; init_php_scripts_in_each_worker(php_script_mutable_globals_singleton, script_task); - auto main_task{DETACH_TASK(std::invoke( - [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { + auto main_task{kphp::coro::detach_task( + [](kphp::coro::detached_task<> script_task) noexcept -> kphp::coro::task<> { // wrap script with additional check for unhandled exception - script_task = DETACH_TASK(std::invoke( - [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { + script_task = kphp::coro::detach_task( + [](kphp::coro::detached_task<> script_task) noexcept -> kphp::coro::task<> { co_await script_task; if (auto exception{std::move(ForkInstanceState::get().current_info().get().thrown_exception)}; !exception.is_null()) [[unlikely]] { kphp::log::error("unhandled exception {}", std::move(exception)); } }, - std::move(script_task))); + std::move(script_task)); kphp::log::assertion(co_await f$wait_concurrently(kphp::forks::start(std::move(script_task)))); }, - std::move(script_task)))}; + std::move(script_task))}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; main_task_async_stack_frame.async_stack_root = std::addressof(coroutine_instance_state.coroutine_stack_root); diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index ce295fc7d6..4f2e475fd7 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -129,7 +129,7 @@ struct InstanceState final : vk::not_copyable { ErrorHandlingState error_handling_instance_state; KmlInstanceState kml_instance_state; - kphp::stl::list, kphp::memory::script_allocator> shutdown_functions; + kphp::stl::list, kphp::memory::script_allocator> shutdown_functions; private: kphp::coro::task<> init_cli_instance() noexcept; diff --git a/runtime-light/core/globals/php-init-scripts.h b/runtime-light/core/globals/php-init-scripts.h index 6c6f3a0046..78515cb168 100644 --- a/runtime-light/core/globals/php-init-scripts.h +++ b/runtime-light/core/globals/php-init-scripts.h @@ -4,11 +4,11 @@ #pragma once -#include "runtime-light/coroutine/task.h" +#include "runtime-light/coroutine/detached-task.h" class PhpScriptMutableGlobals; /// Initializes const variables represented as globals C++ symbols. Definition is generated by compiler. void init_php_scripts_once_in_master() noexcept; /// Initializes mutable globals. Definition is generated by compiler. -void init_php_scripts_in_each_worker(PhpScriptMutableGlobals& php_globals, kphp::coro::task<>& run) noexcept; +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals& php_globals, kphp::coro::detached_task<>& run) noexcept; diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h index dad79eabdf..76756f1a9f 100644 --- a/runtime-light/coroutine/detached-task.h +++ b/runtime-light/coroutine/detached-task.h @@ -19,6 +19,10 @@ namespace kphp::coro { namespace detail { +template +requires std::invocable +class async_function_return_type; + template class awaiter_base { void push_async_stack_frame(async_stack_frame& caller_frame, void* return_address) noexcept { @@ -80,20 +84,23 @@ class awaiter_base { } }; -template -auto make_fork_task(kphp::coro::task&& task) noexcept { - return kphp::coro::detached_task{std::move(task)}; -} - } // namespace detail +template +auto detach_task(F&& f, Args&&... args) noexcept { + auto guard{kphp::coro::task_allocator_guard{}}; + return kphp::coro::detached_task::type>{ + std::invoke(std::forward(f), std::forward(args)...)}; +} + template struct detached_task { private: using task_promise_type = kphp::coro::task::promise_type; explicit detached_task(kphp::coro::task&& task) noexcept - : m_coro(std::exchange(task.m_coro, {})) {} + : m_coro(std::exchange(task.m_coro, {})), + m_stack(m_task_allocator.current_stack()) {} public: detached_task() noexcept = default; @@ -161,10 +168,8 @@ struct detached_task { kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; memory_resource::segmented_stack_resource* m_stack{nullptr}; - template - friend auto detail::make_fork_task(kphp::coro::task&& task) noexcept; + template + friend auto kphp::coro::detach_task(F&& f, Args&&... args) noexcept; }; } // namespace kphp::coro - -#define DETACH_TASK(...) (kphp::coro::task_allocator_guard{}, kphp::coro::detail::make_fork_task(__VA_ARGS__)) diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 9c326513d1..6d3b00058c 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -574,7 +574,8 @@ auto io_scheduler::schedule(coroutine_type coroutine, duration_type timeout) noe } } - auto result{co_await kphp::coro::when_any(std::move(coroutine), DETACH_TASK(make_timeout_task(std::chrono::ceil(timeout))))}; + auto result{co_await kphp::coro::when_any( + std::move(coroutine), kphp::coro::detach_task(&io_scheduler::make_timeout_task, this, std::chrono::ceil(timeout)))}; if (std::holds_alternative(result)) [[unlikely]] { co_return std::unexpected{std::move(std::get<1>(result))}; } diff --git a/runtime-light/server/http/http-server-state.h b/runtime-light/server/http/http-server-state.h index 13d0d1fa94..a8dc989990 100644 --- a/runtime-light/server/http/http-server-state.h +++ b/runtime-light/server/http/http-server-state.h @@ -70,7 +70,7 @@ struct HttpServerInstanceState final : private vk::not_copyable { kphp::http::response_state response_state{kphp::http::response_state::not_started}; // The headers_registered_callback function should only be invoked once - std::optional> headers_registered_callback; + std::optional> headers_registered_callback; kphp::stl::unordered_set, kphp::memory::script_allocator> multipart_temporary_files; diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 8ba88abc71..3f86f1744d 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -110,7 +110,7 @@ class client final { // The protocol design assumes that interrupting the transfer in the middle of a frame leads to critical error. // Therefore, we need to write the request in a separate coroutine. // This technique prevents integrity violations when this coroutine is cancelled. - kphp::coro::io_scheduler::get().start(DETACH_TASK(process_write(t, qid, payload))); + kphp::coro::io_scheduler::get().start(kphp::coro::detach_task(&writer::process_write, this, t, qid, payload)); co_await req_finish_notifier[qid]; kphp::log::assertion(req_status.contains(qid)); @@ -150,7 +150,8 @@ class client final { while (!ctx.get()->interrupted.is_set()) { // Read response header or interrupt - auto read_header_res{co_await kphp::coro::when_any(DETACH_TASK(t.get()->stream.read(resp_header_buf)), interrupter)}; + auto read_header_res{ + co_await kphp::coro::when_any(kphp::coro::detach_task(&kphp::component::stream::read, t.get()->stream, resp_header_buf), interrupter)}; // An interrupt has occurred if (std::holds_alternative(read_header_res)) [[unlikely]] { kphp::log::debug("reader has been interrupted"); diff --git a/runtime-light/stdlib/curl/curl-easy-functions.h b/runtime-light/stdlib/curl/curl-easy-functions.h index 4652fe8b07..b8c8b24c88 100644 --- a/runtime-light/stdlib/curl/curl-easy-functions.h +++ b/runtime-light/stdlib/curl/curl-easy-functions.h @@ -11,6 +11,7 @@ #include #include "runtime-common/core/runtime-core.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/curl/curl-context.h" #include "runtime-light/stdlib/curl/curl-state.h" @@ -475,7 +476,9 @@ inline auto f$curl_exec_concurrently(kphp::web::curl::easy_type easy_id, double auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; auto sched_res{co_await kphp::coro::io_scheduler::get().schedule( - DETACH_TASK(kphp::forks::id_managed(kphp::web::simple::perform, kphp::web::simple::transfer{easy_id})), timeout)}; + kphp::coro::detach_task([](kphp::web::simple::transfer easy_id) noexcept { return kphp::forks::id_managed(kphp::web::simple::perform, easy_id); }, + kphp::web::simple::transfer{easy_id}), + timeout)}; if (!sched_res.has_value()) [[unlikely]] { kphp::web::curl::print_debug( "could not execute curl easy handle concurrently", diff --git a/runtime-light/stdlib/file/resource.h b/runtime-light/stdlib/file/resource.h index cc2f549e92..ee83c32957 100644 --- a/runtime-light/stdlib/file/resource.h +++ b/runtime-light/stdlib/file/resource.h @@ -23,6 +23,7 @@ #include "runtime-common/core/class-instance/refcountable-php-classes.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/k2-platform/k2-api.h" @@ -637,7 +638,7 @@ class socket : public async_resource { socket& operator=(socket&& other) noexcept { if (this != std::addressof(other)) { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(DETACH_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); + kphp::coro::io_scheduler::get().spawn(kphp::coro::detach_task(&socket::finalizer, std::move(m_stream), std::move(m_buf))); } m_open = std::exchange(other.m_open, false); @@ -649,7 +650,7 @@ class socket : public async_resource { ~socket() override { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(DETACH_TASK(finalizer(std::move(m_stream), std::move(m_buf)))); + kphp::coro::io_scheduler::get().spawn(kphp::coro::detach_task(&socket::finalizer, std::move(m_stream), std::move(m_buf))); } } diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index be098a5ef5..af31481b6c 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -16,6 +16,7 @@ #include "runtime-common/core/core-types/decl/optional.h" #include "runtime-common/core/runtime-core.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" @@ -77,7 +78,7 @@ auto id_managed(F f, } template -auto start(kphp::coro::task task) noexcept -> int64_t { +auto start(kphp::coro::detached_task task) noexcept -> int64_t { auto& fork_instance_st{ForkInstanceState::get()}; auto [fork_id, fork_task]{fork_instance_st.create_fork(std::move(task))}; auto saved_fork_id{fork_instance_st.current_id}; diff --git a/runtime-light/stdlib/fork/fork-state.h b/runtime-light/stdlib/fork/fork-state.h index d98eddccb5..d29e38dbba 100644 --- a/runtime-light/stdlib/fork/fork-state.h +++ b/runtime-light/stdlib/fork/fork-state.h @@ -13,8 +13,8 @@ #include "common/mixin/not_copyable.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/shared-task.h" -#include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/exception-types.h" #include "runtime-light/stdlib/diagnostics/logs.h" #include "runtime-light/stdlib/fork/fork-storage.h" @@ -50,9 +50,9 @@ struct ForkInstanceState final : private vk::not_copyable { static ForkInstanceState& get() noexcept; template - std::pair> create_fork(kphp::coro::task task) noexcept { + std::pair> create_fork(kphp::coro::detached_task task) noexcept { static constexpr auto fork_coroutine{ - [](kphp::coro::task task, int64_t fork_id) noexcept -> kphp::coro::shared_task { + [](kphp::coro::detached_task task, int64_t fork_id) noexcept -> kphp::coro::shared_task { ForkInstanceState::get().current_id = fork_id; kphp::forks::details::storage s{}; diff --git a/runtime-light/stdlib/fork/wait-queue-functions.h b/runtime-light/stdlib/fork/wait-queue-functions.h index 4c56bf54b9..74e897996f 100644 --- a/runtime-light/stdlib/fork/wait-queue-functions.h +++ b/runtime-light/stdlib/fork/wait-queue-functions.h @@ -11,6 +11,7 @@ #include "runtime-common/core/core-types/decl/optional.h" #include "runtime-light/coroutine/await-set.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/logs.h" #include "runtime-light/stdlib/fork/fork-functions.h" @@ -56,7 +57,7 @@ inline kphp::coro::task> wait_queue_next(int64_t queue_id timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(DETACH_TASK(wait_queue_next_task(await_set.next())), timeout)}; + auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(kphp::coro::detach_task(wait_queue_next_task, await_set.next()), timeout)}; if (!wait_result) { co_return std::nullopt; } @@ -94,7 +95,7 @@ inline void wait_queue_push(int64_t queue_id, int64_t fork_id) noexcept { }}; auto& await_set{(*opt_await_set).get()}; - await_set.push(DETACH_TASK(wait_queue_wrapper_task(std::move(fork_task), fork_id))); + await_set.push(kphp::coro::detach_task(wait_queue_wrapper_task, std::move(fork_task), fork_id)); } } // namespace kphp::forks diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp index 012dd5c867..fce69b6643 100644 --- a/runtime-light/stdlib/rpc/rpc-api.cpp +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -20,6 +20,7 @@ #include "common/rpc-error-codes.h" #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" @@ -358,7 +359,7 @@ kphp::rpc::query_info send_request(std::string_view actor, std::optional return {reinterpret_cast(response_exp->buffer()), size}; }}; - auto fetch_task{DETACH_TASK(kphp::rpc::query::response(std::move(q), response_buffer_provider))}; + auto fetch_task{kphp::coro::detach_task(kphp::rpc::query::response, std::move(q), response_buffer_provider)}; auto fetch_result{co_await kphp::coro::io_scheduler::get().schedule(std::move(fetch_task))}; if (!fetch_result) [[unlikely]] { response_exp = std::unexpected{fetch_result.error()}; diff --git a/runtime-light/stdlib/rpc/rpc-queue-functions.h b/runtime-light/stdlib/rpc/rpc-queue-functions.h index 0daa7bb0f9..dcf2d961aa 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-functions.h +++ b/runtime-light/stdlib/rpc/rpc-queue-functions.h @@ -43,7 +43,7 @@ inline void rpc_queue_push(int64_t queue_id, int64_t request_id) noexcept { } auto& await_set{(*opt_await_set).get()}; - await_set.push(DETACH_TASK(rpc_queue_wrapper_task(static_cast>(it_awaiter_task->second), request_id))); + await_set.push(kphp::coro::detach_task(rpc_queue_wrapper_task, static_cast>(it_awaiter_task->second), request_id)); } inline int64_t rpc_queue_create(std::span request_ids) noexcept { @@ -83,7 +83,7 @@ inline kphp::coro::task> rpc_queue_next(int64_t queue_id, timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(DETACH_TASK(rpc_queue_next_task(await_set.next())), timeout)}; + const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(kphp::coro::detach_task(rpc_queue_next_task, await_set.next()), timeout)}; if (!expected_next) { co_return std::nullopt; } diff --git a/runtime-light/stdlib/server/handler-functions.h b/runtime-light/stdlib/server/handler-functions.h index 452bac03ab..c8708a3447 100644 --- a/runtime-light/stdlib/server/handler-functions.h +++ b/runtime-light/stdlib/server/handler-functions.h @@ -19,7 +19,7 @@ void f$register_shutdown_function(F&& f, Args&&... args) noexcept { // it's a lambda coroutine, so: // 1. don't capture anything; // 2. parameters are passed by value. - auto shutdown_function_task{DETACH_TASK(std::invoke( + auto shutdown_function_task{kphp::coro::detach_task( [](F f, Args... args) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f), std::move(args)...); @@ -27,7 +27,7 @@ void f$register_shutdown_function(F&& f, Args&&... args) noexcept { std::invoke(std::move(f), std::move(args)...); } }, - std::forward(f), std::forward(args)...))}; + std::forward(f), std::forward(args)...)}; InstanceState::get().shutdown_functions.emplace_back(std::move(shutdown_function_task)); } diff --git a/runtime-light/stdlib/server/http-functions.h b/runtime-light/stdlib/server/http-functions.h index e50c807603..c74f27351d 100644 --- a/runtime-light/stdlib/server/http-functions.h +++ b/runtime-light/stdlib/server/http-functions.h @@ -75,7 +75,7 @@ bool f$header_register_callback(F&& f) noexcept { return false; } - auto headers_callback_task{DETACH_TASK(std::invoke( + auto headers_callback_task{kphp::coro::detach_task( [](F f) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f)); @@ -83,7 +83,7 @@ bool f$header_register_callback(F&& f) noexcept { std::invoke(std::move(f)); } }, - std::forward(f)))}; + std::forward(f))}; http_server_instance_st.headers_registered_callback.emplace(std::move(headers_callback_task)); return true; diff --git a/runtime-light/stdlib/time/timer-functions.h b/runtime-light/stdlib/time/timer-functions.h index e399df98c6..5efa5250cc 100644 --- a/runtime-light/stdlib/time/timer-functions.h +++ b/runtime-light/stdlib/time/timer-functions.h @@ -24,12 +24,12 @@ void f$set_timer(int64_t timeout_ms, T&& on_timer_callback) noexcept { // TODO choose from: // 1. someone should pop that fork from ForkInstanceState since it will stay there unless we perform f$wait on fork // 2. start timer_task via kphp::coro::io_scheduler::spawn (it won't have distinct fork id) - auto timer_task{DETACH_TASK(std::invoke( + auto timer_task{kphp::coro::detach_task( [](std::chrono::milliseconds duration, T on_timer_callback) noexcept -> kphp::coro::task<> { co_await kphp::forks::id_managed([](std::chrono::milliseconds duration) noexcept { return kphp::coro::io_scheduler::get().schedule(duration); }, duration); std::invoke(std::move(on_timer_callback)); }, - std::chrono::milliseconds{timeout_ms}, std::forward(on_timer_callback)))}; + std::chrono::milliseconds{timeout_ms}, std::forward(on_timer_callback))}; kphp::forks::start(std::move(timer_task)); } diff --git a/runtime-light/streams/connection.h b/runtime-light/streams/connection.h index 42762d298e..4e44db20f1 100644 --- a/runtime-light/streams/connection.h +++ b/runtime-light/streams/connection.h @@ -12,6 +12,7 @@ #include #include "runtime-common/core/runtime-core.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -140,7 +141,8 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s } const auto finalizer{vk::finally([state] noexcept { state.get()->m_unwatch_event.reset(); })}; - const auto v{co_await kphp::coro::when_any(DETACH_TASK(unwatch_awaiter(std::move(state))), DETACH_TASK(descriptor_awaiter(descriptor)))}; + const auto v{ + co_await kphp::coro::when_any(kphp::coro::detach_task(unwatch_awaiter, std::move(state)), kphp::coro::detach_task(descriptor_awaiter, descriptor))}; if (std::holds_alternative(v)) { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(h)); @@ -151,7 +153,7 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s }}; m_shared_state.get()->m_unwatch_event.emplace(); - if (!kphp::coro::io_scheduler::get().spawn(DETACH_TASK(watcher(m_stream.descriptor(), m_shared_state, std::forward(h))))) + if (!kphp::coro::io_scheduler::get().spawn(kphp::coro::detach_task(watcher, m_stream.descriptor(), m_shared_state, std::forward(h)))) [[unlikely]] { m_shared_state.get()->m_unwatch_event.reset(); return std::unexpected{k2::errno_ebusy}; From e9f1ce4b6b1663d4d0ecc7836603d05ebb20babc Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 6 Oct 2026 14:45:34 +0300 Subject: [PATCH 09/26] fix bug with dangling references in id_managed --- .../stdlib/component/component-api.h | 11 ++++--- .../stdlib/crypto/crypto-functions.cpp | 33 ++++++++++++------- runtime-light/stdlib/fork/fork-functions.h | 7 ++-- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/runtime-light/stdlib/component/component-api.h b/runtime-light/stdlib/component/component-api.h index acd5839408..4e67d87e12 100644 --- a/runtime-light/stdlib/component/component-api.h +++ b/runtime-light/stdlib/component/component-api.h @@ -119,7 +119,8 @@ inline auto f$component_client_send_request(string name, string request) noexcep auto stream{std::move(*expected_stream)}; auto request_span{std::span{request.c_str(), request.size()}}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::send_request, stream, std::as_bytes(request_span))}; !expected) [[unlikely]] { + if (auto expected{co_await kphp::forks::id_managed(kphp::component::send_request, std::reference_wrapper{stream}, std::as_bytes(request_span))}; !expected) + [[unlikely]] { co_return class_instance{}; } co_return make_instance(std::move(stream)); @@ -133,7 +134,8 @@ inline auto f$component_client_fetch_response(class_instance q string response{}; auto callback{kphp::component::read_ext::append(response)}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_response, query.get()->stream(), std::move(callback))}; + if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_response, std::reference_wrapper{query.get()->stream()}, + std::move(callback))}; !expected) [[unlikely]] { co_return string{}; } @@ -158,7 +160,8 @@ inline auto f$component_server_fetch_request(class_instance qu string request{}; auto callback{kphp::component::read_ext::append(request)}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_request, query.get()->stream(), std::move(callback))}; + if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_request, std::reference_wrapper{query.get()->stream()}, + std::move(callback))}; !expected) [[unlikely]] { co_return string{}; } @@ -173,5 +176,5 @@ inline auto f$component_server_send_response(class_instance qu auto& stream{query.get()->stream()}; auto response_span{std::span{response.c_str(), response.size()}}; - co_await kphp::forks::id_managed(kphp::component::send_response, stream, std::as_bytes(response_span)); + co_await kphp::forks::id_managed(kphp::component::send_response, std::reference_wrapper{stream}, std::as_bytes(response_span)); } diff --git a/runtime-light/stdlib/crypto/crypto-functions.cpp b/runtime-light/stdlib/crypto/crypto-functions.cpp index 8c682a1045..79f468468f 100644 --- a/runtime-light/stdlib/crypto/crypto-functions.cpp +++ b/runtime-light/stdlib/crypto/crypto-functions.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -53,7 +54,8 @@ kphp::coro::task> f$openssl_random_pseudo_bytes(int64_t length) auto stream{*std::move(expected_stream)}; kphp::stl::vector response{}; auto callback{kphp::component::read_ext::append(response)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -77,7 +79,8 @@ kphp::coro::task>> f$openssl_x509_parse(string data, bool auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -125,7 +128,8 @@ kphp::coro::task f$openssl_sign(string data, string& signature, string pri auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -154,7 +158,7 @@ kphp::coro::task f$openssl_verify(string data, string signature, string std::array response{}; if (!co_await kphp::forks::id_managed([](kphp::component::stream& stream, std::span request, std::span response) noexcept { return kphp::component::query(stream, request, response); }, - stream, tls.view(), response)) [[unlikely]] { + std::reference_wrapper{stream}, tls.view(), std::span{response})) [[unlikely]] { co_return 0; } @@ -337,7 +341,8 @@ kphp::coro::task> f$openssl_encrypt(string data, string method, auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -409,7 +414,8 @@ kphp::coro::task> f$openssl_decrypt(string data, string method, auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -436,7 +442,8 @@ kphp::coro::task> f$openssl_pkey_get_public(string key) noexcep auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -466,7 +473,8 @@ kphp::coro::task> f$openssl_pkey_get_private(string key, string auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -496,7 +504,8 @@ kphp::coro::task f$openssl_public_encrypt(string data, string& encrypted_d auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -537,7 +546,8 @@ kphp::coro::task f$openssl_private_decrypt(string data, string& decrypted_ auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -591,7 +601,8 @@ kphp::coro::task send_and_get_string(tl::storer tls, bool raw_output) no auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; auto callback{kphp::component::read_ext::append(response_bytes)}; - if (!co_await kphp::forks::id_managed(kphp::component::query, stream, tls.view(), std::move(callback))) [[unlikely]] { + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index af31481b6c..edb6d83ec2 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -62,16 +62,15 @@ auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task requires(kphp::coro::is_task_function_v) -auto id_managed(F f, - Args&&... args) noexcept -> kphp::coro::task>::awaiter_return_type> { +auto id_managed(F f, Args... args) noexcept -> kphp::coro::task>::awaiter_return_type> { auto& fork_instance_st{ForkInstanceState::get()}; const auto saved_fork_id{fork_instance_st.current_id}; if constexpr (std::is_void_v>::awaiter_return_type>) { - co_await std::invoke(std::move(f), std::forward(args)...); + co_await std::invoke(std::move(f), std::move(args)...); fork_instance_st.current_id = saved_fork_id; co_return; } else { - auto value{co_await std::invoke(std::move(f), std::forward(args)...)}; + auto value{co_await std::invoke(std::move(f), std::move(args)...)}; fork_instance_st.current_id = saved_fork_id; co_return std::move(value); } From 62ce082b95d6ed2596b387e334f74aa1965b0b70 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Wed, 7 Oct 2026 15:08:17 +0300 Subject: [PATCH 10/26] remove redundant branches --- .../memory-resource/chunk-pool-resource.h | 6 ++-- .../segmented-stack-resource.h | 2 -- .../detail/allocator/task-allocator.h | 36 +++++++------------ .../detail/allocator/task-malloc-interface.h | 31 ++-------------- 4 files changed, 18 insertions(+), 57 deletions(-) diff --git a/runtime-common/core/memory-resource/chunk-pool-resource.h b/runtime-common/core/memory-resource/chunk-pool-resource.h index 5970d8dad2..20faccff08 100644 --- a/runtime-common/core/memory-resource/chunk-pool-resource.h +++ b/runtime-common/core/memory-resource/chunk-pool-resource.h @@ -51,8 +51,7 @@ class chunk_pool_resource : private vk::not_copyable { public: auto init(void* buffer, size_t buffer_size, size_t chunk_size) noexcept -> void { - php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= chunk_size + buffer_header_size() && - reinterpret_cast(buffer) % alignof(buffer_list_node) == 0 && chunk_size >= chunk_header_size() && + php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= chunk_size + buffer_header_size() && chunk_size >= chunk_header_size() && chunk_size % alignof(chunk_free_list_node) == 0); m_chunk_size = chunk_size; @@ -85,8 +84,7 @@ class chunk_pool_resource : private vk::not_copyable { } auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { - php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= m_chunk_size + buffer_header_size() && - reinterpret_cast(buffer) % alignof(buffer_list_node) == 0); + php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= m_chunk_size + buffer_header_size()); link_new_buffer(buffer, buffer_size); m_head_buffer = static_cast(buffer); diff --git a/runtime-common/core/memory-resource/segmented-stack-resource.h b/runtime-common/core/memory-resource/segmented-stack-resource.h index 9817c2c6e9..0e0253ca5b 100644 --- a/runtime-common/core/memory-resource/segmented-stack-resource.h +++ b/runtime-common/core/memory-resource/segmented-stack-resource.h @@ -71,8 +71,6 @@ class segmented_stack_resource : private vk::not_copyable { return nullptr; } - php_assert(reinterpret_cast(new_segment) % alignof(segment_list_node) == 0); - switch_to_new_segment(new_segment); } diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h index 4b6474ee72..d3fa05ab2c 100644 --- a/runtime-light/coroutine/detail/allocator/task-allocator.h +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -9,13 +9,20 @@ #include "common/containers/object-pool.h" #include "common/mixin/not_copyable.h" #include "runtime-common/core/allocator/platform-malloc-interface.h" +#include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/memory-resource/chunk-pool-resource.h" #include "runtime-common/core/memory-resource/segmented-stack-resource.h" -#include "runtime-light/coroutine/detail/allocator/coroutine-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro::detail::memory { +namespace task { + +inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void*; +inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void; + +} // namespace task + struct task_allocator final : private vk::not_copyable { struct shared_chunk_pool { memory_resource::chunk_pool_resource& m_chunk_pool; @@ -48,7 +55,7 @@ struct task_allocator final : private vk::not_copyable { private: vk::object_pool, - kphp::coro::detail::memory::coroutine_allocator>> + kphp::memory::script_allocator>> m_stack_pool; memory_resource::chunk_pool_resource m_chunk_pool; memory_resource::segmented_stack_resource* m_curr_stack{nullptr}; @@ -124,10 +131,8 @@ struct task_allocator final : private vk::not_copyable { return prev; } +private: auto alloc_script_memory(size_t size) noexcept -> void* { - kphp::log::assertion(size != 0); - kphp::log::assertion(m_curr_stack != nullptr); - void* mem{m_curr_stack->allocate(size)}; if (mem == nullptr) [[unlikely]] { request_extra_memory(size); @@ -139,27 +144,12 @@ struct task_allocator final : private vk::not_copyable { return mem; } - auto calloc_script_memory(size_t size) noexcept -> void* { - kphp::log::assertion(size != 0); - kphp::log::assertion(m_curr_stack != nullptr); - - void* mem{m_curr_stack->allocate0(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = m_curr_stack->allocate0(size); - - kphp::log::assertion(mem != nullptr); - } - - return mem; - } - auto free_script_memory(void* mem, size_t size) noexcept -> void { - kphp::log::assertion(size != 0); - kphp::log::assertion(m_curr_stack != nullptr); - m_curr_stack->deallocate(mem, size); } + + friend inline auto kphp::coro::detail::memory::task::alloc_aligned(size_t size, std::align_val_t al) noexcept -> void*; + friend inline auto kphp::coro::detail::memory::task::free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void; }; } // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h index 7c50c719b0..13eff701c9 100644 --- a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h +++ b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h @@ -4,7 +4,6 @@ #pragma once -#include #include #include #include @@ -18,9 +17,6 @@ namespace kphp::coro::detail::memory::task { struct control_block { -private: - static constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB - public: enum class backend_type : uint8_t { stack, stack_owner, coroutine_pool }; @@ -28,32 +24,20 @@ struct control_block { uint16_t base_offset{}; backend_type backend{}; - static constexpr auto max_size() noexcept -> uint64_t { - return MALLOC_REPLACER_MAX_ALLOC; - } - static constexpr auto max_alignment() noexcept -> uint64_t { return std::numeric_limits::max(); } }; inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { - // Check that provided alignment is power of two const size_t align{static_cast(al)}; - if (align == 0 || !std::has_single_bit(align) || align >= kphp::coro::detail::memory::task::control_block::max_alignment()) [[unlikely]] { - php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIu64 ", got : %lu", - kphp::coro::detail::memory::task::control_block::max_alignment(), align); + if (align >= kphp::coro::detail::memory::task::control_block::max_alignment()) [[unlikely]] { + php_warning("allocation alignment have to be not greater than %" PRIu64 ", got : %lu", kphp::coro::detail::memory::task::control_block::max_alignment(), + align); return nullptr; } - // Check that memory is enough constexpr size_t cb_size{sizeof(kphp::coro::detail::memory::task::control_block)}; - if (size > kphp::coro::detail::memory::task::control_block::max_size() - (align - 1) - cb_size) [[unlikely]] { - php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size); - return nullptr; - } - - // Request mem from underlying memory manager const size_t total_size{size + (align - 1) + cb_size}; void* base{nullptr}; kphp::coro::detail::memory::task::control_block::backend_type backend{}; @@ -73,11 +57,6 @@ inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { backend = kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool; } - if (base == nullptr) [[unlikely]] { - php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size); - return base; - } - const uint64_t base_u{reinterpret_cast(base)}; // The smallest multiple of `align` greater than or equal to requested memory const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; @@ -90,10 +69,6 @@ inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { } inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void { - if (ptr == nullptr) [[unlikely]] { - return; - } - const size_t align{static_cast(al)}; const size_t cb_size{sizeof(kphp::coro::detail::memory::task::control_block)}; const size_t total_size{size + (align - 1) + cb_size}; From 97fb0584b28ae51745c04b7114d5f4fae444393a Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Thu, 8 Oct 2026 18:44:20 +0300 Subject: [PATCH 11/26] add stack cache --- .../segmented-stack-resource.h | 20 +++++++++------- .../detail/allocator/task-allocator.h | 24 +++++++++++++++---- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/runtime-common/core/memory-resource/segmented-stack-resource.h b/runtime-common/core/memory-resource/segmented-stack-resource.h index 0e0253ca5b..f20143051f 100644 --- a/runtime-common/core/memory-resource/segmented-stack-resource.h +++ b/runtime-common/core/memory-resource/segmented-stack-resource.h @@ -43,13 +43,8 @@ class segmented_stack_resource : private vk::not_copyable { m_head = m_head->m_next; m_segment_pool.deallocate(top_segment); - if (m_head != nullptr) { - m_segment_begin = reinterpret_cast(m_head) + segment_header_size(); - m_segment_curr = m_head->m_curr; - } else { - m_segment_begin = nullptr; - m_segment_curr = nullptr; - } + m_segment_begin = reinterpret_cast(m_head) + segment_header_size(); + m_segment_curr = m_head->m_curr; } public: @@ -58,6 +53,13 @@ class segmented_stack_resource : private vk::not_copyable { m_segment_size = segment_size; m_segment_pool.init(buffer, buffer_size, m_segment_size + segment_header_size()); + + void* segment{m_segment_pool.allocate()}; + if (unlikely(segment == nullptr)) { + return; + } + + switch_to_new_segment(segment); } auto allocate(size_t size) noexcept -> void* { @@ -65,7 +67,7 @@ class segmented_stack_resource : private vk::not_copyable { return nullptr; } - if (m_head == nullptr || size > m_segment_size - static_cast(m_segment_curr - m_segment_begin)) { + if (unlikely(m_head == nullptr || size > m_segment_size - static_cast(m_segment_curr - m_segment_begin))) { void* new_segment{m_segment_pool.allocate()}; if (unlikely(new_segment == nullptr)) { return nullptr; @@ -94,7 +96,7 @@ class segmented_stack_resource : private vk::not_copyable { php_assert(static_cast(mem) == m_segment_curr); - if (m_segment_curr == m_segment_begin) { + if (m_segment_curr == m_segment_begin && m_head->m_next != nullptr) { switch_to_old_segment(); } } diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h index d3fa05ab2c..c30bc762ff 100644 --- a/runtime-light/coroutine/detail/allocator/task-allocator.h +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -12,6 +12,7 @@ #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/memory-resource/chunk-pool-resource.h" #include "runtime-common/core/memory-resource/segmented-stack-resource.h" +#include "runtime-common/core/std/containers.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro::detail::memory { @@ -58,6 +59,7 @@ struct task_allocator final : private vk::not_copyable { kphp::memory::script_allocator>> m_stack_pool; memory_resource::chunk_pool_resource m_chunk_pool; + kphp::stl::vector*, kphp::memory::script_allocator> m_free_stacks; memory_resource::segmented_stack_resource* m_curr_stack{nullptr}; size_t m_segment_size{0}; size_t m_min_extra_mem_size{0}; @@ -88,9 +90,14 @@ struct task_allocator final : private vk::not_copyable { kphp::log::assertion(buffer != nullptr); m_chunk_pool.init(buffer, script_mem_size, m_segment_size + memory_resource::segmented_stack_resource::segment_header_size()); + m_free_stacks.reserve(stack_pool_chunk_size); } auto free() noexcept -> void { + for (auto* stack : m_free_stacks) { + m_stack_pool.release(*stack); + } + auto* curr_buffer{m_chunk_pool.get_buffer_list_head()}; while (curr_buffer != nullptr) { auto* next_buffer = curr_buffer->next; @@ -100,15 +107,22 @@ struct task_allocator final : private vk::not_copyable { } auto acquire_stack() noexcept -> memory_resource::segmented_stack_resource& { - auto& stack{m_stack_pool.acquire()}; - // We can pass nullptr as buffer and 0 as buffer_size, because init in shared_chunk_pool is noop - stack.init(nullptr, 0, m_segment_size); + if (m_free_stacks.empty()) { + auto& stack{m_stack_pool.acquire()}; + // We can pass nullptr as buffer and 0 as buffer_size, because init in shared_chunk_pool is noop + stack.init(nullptr, 0, m_segment_size); + + return stack; + } + + auto* stack{m_free_stacks.back()}; + m_free_stacks.pop_back(); - return stack; + return *stack; } auto release_stack(memory_resource::segmented_stack_resource& stack) noexcept -> void { - m_stack_pool.release(stack); + m_free_stacks.push_back(std::addressof(stack)); } auto segment_size() const noexcept -> size_t { From e41a89ccf460a31bb20c6927f0ba45987a50eec4 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Fri, 9 Oct 2026 16:57:00 +0300 Subject: [PATCH 12/26] refactor control functions --- runtime-light/coroutine/control-functions.h | 11 +++++++---- runtime-light/coroutine/task.h | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/runtime-light/coroutine/control-functions.h b/runtime-light/coroutine/control-functions.h index d7529089a5..cdc8c46955 100644 --- a/runtime-light/coroutine/control-functions.h +++ b/runtime-light/coroutine/control-functions.h @@ -20,26 +20,29 @@ namespace kphp::coro { * All calls to resume() method of handle must be made with overload of this function. */ inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - kphp::coro::task_allocator_guard guard{task_allocator}; + auto* prev_stack{task_allocator.current_stack()}; auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; handle.resume(); stack_root.stop_sync_stack_frame = previous_stack_frame; + task_allocator.set_stack(prev_stack); } /* * All calls to resume() method of handle must be made with overload of this function. */ inline void resume(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - kphp::coro::task_allocator_guard guard{task_allocator}; + auto* prev_stack{task_allocator.current_stack()}; handle.resume(); + task_allocator.set_stack(prev_stack); } /* - * All calls to destroy() method of handle must be made with this function (except for task destructor). + * All calls to destroy() method of handle must be made with this function (except for detached_task destructor). */ inline void destroy(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - kphp::coro::task_allocator_guard guard{task_allocator}; + auto* prev_stack{task_allocator.current_stack()}; handle.destroy(); + task_allocator.set_stack(prev_stack); } } // namespace kphp::coro diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index f56485c2d9..d223404ead 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -12,6 +12,8 @@ #include "common/containers/final_action.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/detail/allocator/task-malloc-interface.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -167,7 +169,7 @@ struct task { ~task() { if (m_coro) { - m_coro.destroy(); + kphp::coro::destroy(m_coro, kphp::coro::detail::memory::task_allocator::get()); } } From d36afa4eb4d9377f92d1c53784b2fb534fe0fd8b Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Fri, 9 Oct 2026 16:58:18 +0300 Subject: [PATCH 13/26] refactor control functions --- runtime-light/coroutine/control-functions.h | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime-light/coroutine/control-functions.h b/runtime-light/coroutine/control-functions.h index cdc8c46955..2d882a788d 100644 --- a/runtime-light/coroutine/control-functions.h +++ b/runtime-light/coroutine/control-functions.h @@ -9,7 +9,6 @@ #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" -#include "runtime-light/coroutine/task-allocator-guard.h" namespace kphp::coro { From 96ef18eae2fc7e34c7ff54669370273fa41d44f1 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Fri, 9 Oct 2026 17:13:19 +0300 Subject: [PATCH 14/26] refactor object pool --- common/containers/object-pool.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/common/containers/object-pool.h b/common/containers/object-pool.h index a56982472f..0abb780885 100644 --- a/common/containers/object-pool.h +++ b/common/containers/object-pool.h @@ -19,33 +19,33 @@ class object_pool : private vk::not_copyable, private std::allocator_traits::template rebind_alloc; using byte_allocator_traits = std::allocator_traits; - union object_pool_slot { - object_pool_slot* m_next; + union obj_slot { + obj_slot* m_next; T m_obj; }; - struct alignas(alignof(object_pool_slot)) object_pool_chunk_header { - object_pool_chunk_header* m_next{nullptr}; + struct alignas(alignof(obj_slot)) chunk_header { + chunk_header* m_next{nullptr}; }; std::size_t m_chunk_size{0}; std::size_t m_chunk_byte_size{0}; - object_pool_chunk_header* m_head_chunk{nullptr}; - object_pool_slot* m_head_free_slot{nullptr}; + chunk_header* m_head_chunk{nullptr}; + obj_slot* m_head_free_slot{nullptr}; auto get_byte_allocator_ref() noexcept -> byte_allocator& { return *this; } - static auto slots_of(object_pool_chunk_header* chunk) noexcept -> object_pool_slot* { - return reinterpret_cast(reinterpret_cast(chunk) + sizeof(object_pool_chunk_header)); + static auto slots_of(chunk_header* chunk) noexcept -> obj_slot* { + return reinterpret_cast(reinterpret_cast(chunk) + sizeof(chunk_header)); } auto link_new_chunk() noexcept -> void { std::byte* mem{byte_allocator_traits::allocate(get_byte_allocator_ref(), m_chunk_byte_size)}; - m_head_chunk = new (mem) object_pool_chunk_header{m_head_chunk}; + m_head_chunk = new (mem) chunk_header{m_head_chunk}; - object_pool_slot* slots{slots_of(m_head_chunk)}; + obj_slot* slots{slots_of(m_head_chunk)}; slots[0].m_next = nullptr; for (size_t i = 1; i < m_chunk_size; ++i) { slots[i].m_next = std::addressof(slots[i - 1]); @@ -57,7 +57,7 @@ class object_pool : private vk::not_copyable, private std::allocator_traits 0); link_new_chunk(); @@ -66,7 +66,7 @@ class object_pool : private vk::not_copyable, private std::allocator_traits 0); link_new_chunk(); @@ -89,7 +89,7 @@ class object_pool : private vk::not_copyable, private std::allocator_traitsm_next; byte_allocator_traits::construct(get_byte_allocator_ref(), std::addressof(free_slot->m_obj), std::forward(args)...); @@ -100,7 +100,7 @@ class object_pool : private vk::not_copyable, private std::allocator_traits void { byte_allocator_traits::destroy(get_byte_allocator_ref(), std::addressof(obj)); - auto* slot{reinterpret_cast(std::addressof(obj))}; + auto* slot{reinterpret_cast(std::addressof(obj))}; slot->m_next = m_head_free_slot; m_head_free_slot = slot; } From cd71f991f2cc8f732bf17975c637e644ba6688ca Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 12 Oct 2026 14:20:14 +0300 Subject: [PATCH 15/26] reduce number of allocator switches --- runtime-light/coroutine/control-functions.h | 38 ++++++++++++------- runtime-light/coroutine/detached-task.h | 13 +++---- runtime-light/coroutine/shared-task.h | 3 +- .../coroutine/task-allocator-guard.h | 11 ++---- .../coroutine/task-allocator-scope.h | 32 ++++++++++++++++ 5 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 runtime-light/coroutine/task-allocator-scope.h diff --git a/runtime-light/coroutine/control-functions.h b/runtime-light/coroutine/control-functions.h index 2d882a788d..164d3b09f7 100644 --- a/runtime-light/coroutine/control-functions.h +++ b/runtime-light/coroutine/control-functions.h @@ -9,6 +9,8 @@ #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" +#include "runtime-light/coroutine/task-allocator-scope.h" namespace kphp::coro { @@ -16,32 +18,42 @@ namespace kphp::coro { * The `resume` function is responsible for storing the current synchronous stack frame * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows * capturing one of the stack frames in the synchronous stack trace. - * All calls to resume() method of handle must be made with overload of this function. */ inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - auto* prev_stack{task_allocator.current_stack()}; + kphp::coro::task_allocator_guard guard{task_allocator}; auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; handle.resume(); stack_root.stop_sync_stack_frame = previous_stack_frame; - task_allocator.set_stack(prev_stack); } -/* - * All calls to resume() method of handle must be made with overload of this function. - */ inline void resume(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - auto* prev_stack{task_allocator.current_stack()}; + kphp::coro::task_allocator_guard guard{task_allocator}; handle.resume(); - task_allocator.set_stack(prev_stack); } -/* - * All calls to destroy() method of handle must be made with this function (except for detached_task destructor). - */ inline void destroy(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - auto* prev_stack{task_allocator.current_stack()}; + kphp::coro::task_allocator_guard guard{task_allocator}; + handle.destroy(); +} + +inline void resume_with_stack(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::segmented_stack_resource* stack) noexcept { + kphp::coro::task_allocator_scope scope{task_allocator, stack}; + auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; + handle.resume(); + stack_root.stop_sync_stack_frame = previous_stack_frame; +} + +inline void resume_with_stack(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::segmented_stack_resource* stack) noexcept { + kphp::coro::task_allocator_scope scope{task_allocator, stack}; + handle.resume(); +} + +inline void destroy_with_stack(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::segmented_stack_resource* stack) noexcept { + kphp::coro::task_allocator_scope scope{task_allocator, stack}; handle.destroy(); - task_allocator.set_stack(prev_stack); } } // namespace kphp::coro diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h index 76756f1a9f..31cc929a41 100644 --- a/runtime-light/coroutine/detached-task.h +++ b/runtime-light/coroutine/detached-task.h @@ -10,8 +10,9 @@ #include #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" -#include "runtime-light/coroutine/task-allocator-guard.h" +#include "runtime-light/coroutine/task-allocator-scope.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -88,7 +89,7 @@ class awaiter_base { template auto detach_task(F&& f, Args&&... args) noexcept { - auto guard{kphp::coro::task_allocator_guard{}}; + auto scope{kphp::coro::task_allocator_scope{nullptr}}; return kphp::coro::detached_task::type>{ std::invoke(std::forward(f), std::forward(args)...)}; } @@ -122,19 +123,17 @@ struct detached_task { ~detached_task() { if (m_coro) { - auto* prev_stack{m_task_allocator.exchange_stack(m_stack)}; - m_coro.destroy(); - m_task_allocator.set_stack(prev_stack); + kphp::coro::destroy_with_stack(m_coro, m_task_allocator, m_stack); } } constexpr auto operator co_await() noexcept { using awaiter_base = detail::awaiter_base; - struct awaiter final : public awaiter_base, private kphp::coro::task_allocator_guard { + struct awaiter final : public awaiter_base, private kphp::coro::task_allocator_scope { awaiter(std::coroutine_handle coro, kphp::coro::detail::memory::task_allocator& task_allocator, memory_resource::segmented_stack_resource* stack) noexcept : awaiter_base(coro), - kphp::coro::task_allocator_guard(task_allocator, stack) {} + kphp::coro::task_allocator_scope(task_allocator, stack) {} auto await_resume() noexcept -> T { awaiter_base::await_resume(); diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index e85413fa58..5d3148f036 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -21,6 +21,7 @@ #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/task-allocator-guard.h" +#include "runtime-light/coroutine/task-allocator-scope.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -106,7 +107,7 @@ struct promise_base : public kphp::coro::async_stack_element { m_state = vk::intrusive::list>>{}; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto& async_stack_root{*get_async_stack_frame().async_stack_root}; - kphp::coro::resume(handle, async_stack_root, m_task_allocator); + kphp::coro::resume_with_stack(handle, async_stack_root, m_task_allocator, nullptr); } // coroutine already completed, don't suspend diff --git a/runtime-light/coroutine/task-allocator-guard.h b/runtime-light/coroutine/task-allocator-guard.h index d770ee0470..ce70c519d6 100644 --- a/runtime-light/coroutine/task-allocator-guard.h +++ b/runtime-light/coroutine/task-allocator-guard.h @@ -16,15 +16,12 @@ class task_allocator_guard : private vk::not_copyable { memory_resource::segmented_stack_resource* m_stack{nullptr}; public: - explicit task_allocator_guard( - memory_resource::segmented_stack_resource* new_stack = nullptr) noexcept - : m_stack{m_task_allocator.exchange_stack(new_stack)} {} + explicit task_allocator_guard() noexcept + : m_stack{m_task_allocator.current_stack()} {} - explicit task_allocator_guard( - kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* new_stack = nullptr) noexcept + explicit task_allocator_guard(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept : m_task_allocator{task_allocator}, - m_stack{m_task_allocator.exchange_stack(new_stack)} {} + m_stack{m_task_allocator.current_stack()} {} ~task_allocator_guard() { m_task_allocator.set_stack(m_stack); diff --git a/runtime-light/coroutine/task-allocator-scope.h b/runtime-light/coroutine/task-allocator-scope.h new file mode 100644 index 0000000000..549ca27cf1 --- /dev/null +++ b/runtime-light/coroutine/task-allocator-scope.h @@ -0,0 +1,32 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/segmented-stack-resource.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" + +namespace kphp::coro { + +class task_allocator_scope : private vk::not_copyable { +private: + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + memory_resource::segmented_stack_resource* m_stack{nullptr}; + +public: + explicit task_allocator_scope(memory_resource::segmented_stack_resource* new_stack) noexcept + : m_stack{m_task_allocator.exchange_stack(new_stack)} {} + + explicit task_allocator_scope(kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::segmented_stack_resource* new_stack) noexcept + : m_task_allocator{task_allocator}, + m_stack{m_task_allocator.exchange_stack(new_stack)} {} + + ~task_allocator_scope() { + m_task_allocator.set_stack(m_stack); + } +}; + +} // namespace kphp::coro From c043364cd5cfd34aa7372b44a872eaad18c35430 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 12 Oct 2026 14:20:38 +0300 Subject: [PATCH 16/26] reduce number of allocator switches --- runtime-light/coroutine/shared-task.h | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 5d3148f036..7bdc52e2d0 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -21,7 +21,6 @@ #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/task-allocator-guard.h" -#include "runtime-light/coroutine/task-allocator-scope.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" From 7a0c734c222440936e164243f903ee180e8ca75f Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 12 Oct 2026 14:24:37 +0300 Subject: [PATCH 17/26] reduce number of allocator switches --- runtime-light/coroutine/detail/await-set.h | 3 ++- runtime-light/coroutine/detail/when-all.h | 2 +- runtime-light/coroutine/detail/when-any.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index c5f315afbf..77cf0f691a 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -245,7 +245,8 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_await_broker->get().task_allocator()); + kphp::coro::resume_with_stack(std::coroutine_handle::from_promise(*static_cast(this)), m_await_broker->get().task_allocator(), + nullptr); } }; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index c214e0cfe1..e4139a8b25 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -219,7 +219,7 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), latch.task_allocator()); + kphp::coro::resume_with_stack(std::coroutine_handle::from_promise(*static_cast(this)), latch.task_allocator(), nullptr); } }; diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 00be6508a4..a4662a2a50 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -228,7 +228,7 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_latch->task_allocator()); + kphp::coro::resume_with_stack(std::coroutine_handle::from_promise(*static_cast(this)), m_latch->task_allocator(), nullptr); } }; From 0dd2bce9d92887ab699dc55d94a32ec0d6aec367 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 12 Oct 2026 14:53:06 +0300 Subject: [PATCH 18/26] reduce number of allocator switches --- runtime-light/coroutine/control-functions.h | 27 +++---------------- runtime-light/coroutine/detached-task.h | 3 ++- runtime-light/coroutine/detail/await-set.h | 3 +-- runtime-light/coroutine/detail/when-all.h | 2 +- runtime-light/coroutine/detail/when-any.h | 2 +- runtime-light/coroutine/shared-task.h | 2 +- .../coroutine/task-allocator-scope.h | 8 +++--- runtime-light/coroutine/task.h | 3 ++- 8 files changed, 15 insertions(+), 35 deletions(-) diff --git a/runtime-light/coroutine/control-functions.h b/runtime-light/coroutine/control-functions.h index 164d3b09f7..26cb9ce96b 100644 --- a/runtime-light/coroutine/control-functions.h +++ b/runtime-light/coroutine/control-functions.h @@ -9,7 +9,6 @@ #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" -#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/task-allocator-scope.h" namespace kphp::coro { @@ -20,39 +19,19 @@ namespace kphp::coro { * capturing one of the stack frames in the synchronous stack trace. */ inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - kphp::coro::task_allocator_guard guard{task_allocator}; + kphp::coro::task_allocator_scope scope{task_allocator, nullptr}; auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; handle.resume(); stack_root.stop_sync_stack_frame = previous_stack_frame; } inline void resume(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - kphp::coro::task_allocator_guard guard{task_allocator}; + kphp::coro::task_allocator_scope scope{task_allocator, nullptr}; handle.resume(); } inline void destroy(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { - kphp::coro::task_allocator_guard guard{task_allocator}; - handle.destroy(); -} - -inline void resume_with_stack(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* stack) noexcept { - kphp::coro::task_allocator_scope scope{task_allocator, stack}; - auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; - handle.resume(); - stack_root.stop_sync_stack_frame = previous_stack_frame; -} - -inline void resume_with_stack(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* stack) noexcept { - kphp::coro::task_allocator_scope scope{task_allocator, stack}; - handle.resume(); -} - -inline void destroy_with_stack(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* stack) noexcept { - kphp::coro::task_allocator_scope scope{task_allocator, stack}; + kphp::coro::task_allocator_scope scope{task_allocator, nullptr}; handle.destroy(); } diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h index 31cc929a41..4aca994d3b 100644 --- a/runtime-light/coroutine/detached-task.h +++ b/runtime-light/coroutine/detached-task.h @@ -123,7 +123,8 @@ struct detached_task { ~detached_task() { if (m_coro) { - kphp::coro::destroy_with_stack(m_coro, m_task_allocator, m_stack); + m_task_allocator.set_stack(m_stack); + m_coro.destroy(); } } diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 77cf0f691a..c5f315afbf 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -245,8 +245,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - kphp::coro::resume_with_stack(std::coroutine_handle::from_promise(*static_cast(this)), m_await_broker->get().task_allocator(), - nullptr); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_await_broker->get().task_allocator()); } }; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index e4139a8b25..c214e0cfe1 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -219,7 +219,7 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - kphp::coro::resume_with_stack(std::coroutine_handle::from_promise(*static_cast(this)), latch.task_allocator(), nullptr); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), latch.task_allocator()); } }; diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index a4662a2a50..00be6508a4 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -228,7 +228,7 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - kphp::coro::resume_with_stack(std::coroutine_handle::from_promise(*static_cast(this)), m_latch->task_allocator(), nullptr); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_latch->task_allocator()); } }; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 7bdc52e2d0..e85413fa58 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -106,7 +106,7 @@ struct promise_base : public kphp::coro::async_stack_element { m_state = vk::intrusive::list>>{}; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto& async_stack_root{*get_async_stack_frame().async_stack_root}; - kphp::coro::resume_with_stack(handle, async_stack_root, m_task_allocator, nullptr); + kphp::coro::resume(handle, async_stack_root, m_task_allocator); } // coroutine already completed, don't suspend diff --git a/runtime-light/coroutine/task-allocator-scope.h b/runtime-light/coroutine/task-allocator-scope.h index 549ca27cf1..c3384a2e51 100644 --- a/runtime-light/coroutine/task-allocator-scope.h +++ b/runtime-light/coroutine/task-allocator-scope.h @@ -16,13 +16,13 @@ class task_allocator_scope : private vk::not_copyable { memory_resource::segmented_stack_resource* m_stack{nullptr}; public: - explicit task_allocator_scope(memory_resource::segmented_stack_resource* new_stack) noexcept - : m_stack{m_task_allocator.exchange_stack(new_stack)} {} + explicit task_allocator_scope(memory_resource::segmented_stack_resource* stack) noexcept + : m_stack{m_task_allocator.exchange_stack(stack)} {} explicit task_allocator_scope(kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* new_stack) noexcept + memory_resource::segmented_stack_resource* stack) noexcept : m_task_allocator{task_allocator}, - m_stack{m_task_allocator.exchange_stack(new_stack)} {} + m_stack{m_task_allocator.exchange_stack(stack)} {} ~task_allocator_scope() { m_task_allocator.set_stack(m_stack); diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index d223404ead..54bf27d035 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -15,6 +15,7 @@ #include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/detail/allocator/task-malloc-interface.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { @@ -169,7 +170,7 @@ struct task { ~task() { if (m_coro) { - kphp::coro::destroy(m_coro, kphp::coro::detail::memory::task_allocator::get()); + m_coro.destroy(); } } From 5b0cbb36e6f2cf21589bf3d3c2aa5d6ed0fa5391 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 12 Oct 2026 15:33:30 +0300 Subject: [PATCH 19/26] fix bug with allocator switching in detached-task destructor --- runtime-light/coroutine/detached-task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h index 4aca994d3b..374b5ce19e 100644 --- a/runtime-light/coroutine/detached-task.h +++ b/runtime-light/coroutine/detached-task.h @@ -123,7 +123,7 @@ struct detached_task { ~detached_task() { if (m_coro) { - m_task_allocator.set_stack(m_stack); + kphp::coro::task_allocator_scope scope{m_task_allocator, m_stack}; m_coro.destroy(); } } From c3057b1576991ae2cb0a34d6012e37b77fa80c5b Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Fri, 16 Oct 2026 17:10:50 +0300 Subject: [PATCH 20/26] add stack resource --- .../segmented-stack-resource.h | 4 +- .../core/memory-resource/stack-resource.h | 60 +++++++ .../segmented-stack-resource-test.cpp | 2 - .../memory_resource/stack-resource-test.cpp | 170 ++++++++++++++++++ tests/cpp/runtime/runtime-tests.cmake | 1 + 5 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 runtime-common/core/memory-resource/stack-resource.h create mode 100644 tests/cpp/runtime/memory_resource/stack-resource-test.cpp diff --git a/runtime-common/core/memory-resource/segmented-stack-resource.h b/runtime-common/core/memory-resource/segmented-stack-resource.h index f20143051f..a2a2e81ab4 100644 --- a/runtime-common/core/memory-resource/segmented-stack-resource.h +++ b/runtime-common/core/memory-resource/segmented-stack-resource.h @@ -92,9 +92,9 @@ class segmented_stack_resource : private vk::not_copyable { } auto deallocate(void* mem, size_t size) noexcept -> void { - m_segment_curr -= size; + php_assert(m_segment_curr - static_cast(mem) == size); - php_assert(static_cast(mem) == m_segment_curr); + m_segment_curr -= size; if (m_segment_curr == m_segment_begin && m_head->m_next != nullptr) { switch_to_old_segment(); diff --git a/runtime-common/core/memory-resource/stack-resource.h b/runtime-common/core/memory-resource/stack-resource.h new file mode 100644 index 0000000000..04d1b12da1 --- /dev/null +++ b/runtime-common/core/memory-resource/stack-resource.h @@ -0,0 +1,60 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/memory_resource.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace memory_resource { + +class stack_resource : private vk::not_copyable { + std::byte* m_begin{nullptr}; + std::byte* m_curr{nullptr}; + std::byte* m_end{nullptr}; + +public: + auto init(void* buffer, size_t buffer_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit()); + + m_begin = static_cast(buffer); + m_curr = m_begin; + m_end = m_curr + buffer_size; + } + + auto allocate(size_t size) noexcept -> void* { + if (size > static_cast(m_end - m_curr)) { + return nullptr; + } + + void* allocated{m_curr}; + m_curr += size; + + return allocated; + } + + auto allocate0(size_t size) noexcept -> void* { + void* allocated{allocate(size)}; + if (allocated != nullptr) { + memset(allocated, 0x00, size); + } + + return allocated; + } + + auto deallocate(void* mem, size_t size) noexcept -> void { + php_assert(m_curr - static_cast(mem) == size); + m_curr -= size; + } + + auto memory_begin() noexcept -> void* { + return m_begin; + } +}; + +} // namespace memory_resource diff --git a/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp b/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp index 99abee47a3..c5fbad7b89 100644 --- a/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp +++ b/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp @@ -36,8 +36,6 @@ auto segment_payload(std::byte* buffer_start, size_t index) noexcept -> std::byt TEST(segmented_stack_resource_test, uninited_state) { memory_resource::segmented_stack_resource resource; - ASSERT_EQ(resource.allocate(1), nullptr); - ASSERT_EQ(resource.allocate0(1), nullptr); ASSERT_EQ(resource.get_buffer_list_head(), nullptr); } diff --git a/tests/cpp/runtime/memory_resource/stack-resource-test.cpp b/tests/cpp/runtime/memory_resource/stack-resource-test.cpp new file mode 100644 index 0000000000..18e00b7be8 --- /dev/null +++ b/tests/cpp/runtime/memory_resource/stack-resource-test.cpp @@ -0,0 +1,170 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include + +#include + +#include "runtime-common/core/memory-resource/stack-resource.h" + +namespace { + +constexpr size_t BUFFER_SIZE{64}; + +} // namespace + +TEST(stack_resource_test, uninited_state) { + memory_resource::stack_resource resource; + + ASSERT_EQ(resource.memory_begin(), nullptr); +} + +TEST(stack_resource_test, single_allocation_exact_buffer_fit) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* mem{resource.allocate(BUFFER_SIZE)}; + + ASSERT_EQ(mem, static_cast(buffer.data())); + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(stack_resource_test, sequential_allocations_bump_pointer) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* a{resource.allocate(16)}; + void* b{resource.allocate(16)}; + void* c{resource.allocate(32)}; + + ASSERT_EQ(a, static_cast(buffer.data())); + ASSERT_EQ(b, static_cast(buffer.data() + 16)); + ASSERT_EQ(c, static_cast(buffer.data() + 32)); + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(stack_resource_test, allocation_exceeding_remaining_capacity_returns_null_without_corrupting_state) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* mem{resource.allocate(BUFFER_SIZE - 16)}; + + ASSERT_NE(mem, nullptr); + ASSERT_EQ(resource.allocate(32), nullptr); + + void* rest{resource.allocate(16)}; + + ASSERT_EQ(rest, static_cast(buffer.data() + (BUFFER_SIZE - 16))); + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(stack_resource_test, lifo_deallocate_reuses_address) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + resource.allocate(16); + void* b1{resource.allocate(16)}; + + resource.deallocate(b1, 16); + void* b2{resource.allocate(16)}; + + ASSERT_EQ(b2, b1); +} + +TEST(stack_resource_test, multiple_lifo_deallocations_restore_cursor_in_order) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* a{resource.allocate(16)}; + void* b{resource.allocate(16)}; + void* c{resource.allocate(16)}; + + resource.deallocate(c, 16); + resource.deallocate(b, 16); + + void* b_again{resource.allocate(16)}; + void* c_again{resource.allocate(16)}; + + ASSERT_EQ(b_again, b); + ASSERT_EQ(c_again, c); + + resource.deallocate(c_again, 16); + resource.deallocate(b_again, 16); + resource.deallocate(a, 16); + + ASSERT_EQ(resource.allocate(BUFFER_SIZE), static_cast(buffer.data())); +} + +TEST(stack_resource_test, allocate0_returns_zeroed_memory) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* mem{resource.allocate(BUFFER_SIZE)}; + ASSERT_NE(mem, nullptr); + std::memset(mem, 0xAB, BUFFER_SIZE); + resource.deallocate(mem, BUFFER_SIZE); + + void* mem0{resource.allocate0(BUFFER_SIZE)}; + + ASSERT_EQ(mem0, mem); + + auto* bytes{static_cast(mem0)}; + for (size_t i = 0; i < BUFFER_SIZE; ++i) { + ASSERT_EQ(static_cast(bytes[i]), 0); + } +} + +TEST(stack_resource_test, allocate0_on_full_buffer_returns_null) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + ASSERT_NE(resource.allocate(BUFFER_SIZE), nullptr); + ASSERT_EQ(resource.allocate0(1), nullptr); +} + +TEST(stack_resource_test, allocations_do_not_overlap) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* a{resource.allocate(20)}; + void* b{resource.allocate(20)}; + void* c{resource.allocate(20)}; + + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + ASSERT_NE(c, nullptr); + + std::memset(a, 1, 20); + std::memset(b, 2, 20); + std::memset(c, 2, 20); // intentionally same fill as b, to catch an overlap via pointer identity instead + + ASSERT_NE(a, b); + ASSERT_NE(b, c); + ASSERT_EQ(static_cast(b) - static_cast(a), 20); + ASSERT_EQ(static_cast(c) - static_cast(b), 20); +} + +TEST(stack_resource_test, memory_begin_returns_initial_buffer_and_is_stable) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + ASSERT_EQ(resource.memory_begin(), static_cast(buffer.data())); + + void* mem{resource.allocate(16)}; + ASSERT_EQ(resource.memory_begin(), static_cast(buffer.data())); + + resource.deallocate(mem, 16); + ASSERT_EQ(resource.memory_begin(), static_cast(buffer.data())); +} diff --git a/tests/cpp/runtime/runtime-tests.cmake b/tests/cpp/runtime/runtime-tests.cmake index d403b5e385..bf413fd41a 100644 --- a/tests/cpp/runtime/runtime-tests.cmake +++ b/tests/cpp/runtime/runtime-tests.cmake @@ -20,6 +20,7 @@ prepend(RUNTIME_TESTS_SOURCES ${BASE_DIR}/tests/cpp/runtime/ memory_resource/details/memory_ordered_chunk_list-test.cpp memory_resource/extra-memory-pool-test.cpp memory_resource/segmented-stack-resource-test.cpp + memory_resource/stack-resource-test.cpp memory_resource/unsynchronized_pool_resource-test.cpp string-list-test.cpp string-test.cpp From 6f43cc2155b7143e928b1089840da9d2fd45c742 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 20 Oct 2026 15:02:48 +0300 Subject: [PATCH 21/26] rewrite task allocator to use stack resource --- common/containers/object-pool.h | 12 +- .../segmented-stack-resource.h | 117 ------- .../core/memory-resource/stack-resource.h | 4 + .../confdata/state/component-state.h | 4 +- .../confdata/state/instance-state.h | 2 +- .../components/kphp/state/component-state.cpp | 8 +- .../components/kphp/state/component-state.h | 6 +- .../components/kphp/state/instance-state.h | 2 +- runtime-light/coroutine/coroutine-state.h | 4 +- runtime-light/coroutine/detached-task.h | 4 +- .../detail/allocator/task-allocator.h | 112 ++----- .../detail/allocator/task-malloc-interface.h | 12 +- .../coroutine/task-allocator-guard.h | 4 +- .../coroutine/task-allocator-scope.h | 9 +- .../segmented-stack-resource-test.cpp | 287 ------------------ tests/cpp/runtime/runtime-tests.cmake | 1 - 16 files changed, 71 insertions(+), 517 deletions(-) delete mode 100644 runtime-common/core/memory-resource/segmented-stack-resource.h delete mode 100644 tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp diff --git a/common/containers/object-pool.h b/common/containers/object-pool.h index 0abb780885..3e0a89389c 100644 --- a/common/containers/object-pool.h +++ b/common/containers/object-pool.h @@ -28,8 +28,8 @@ class object_pool : private vk::not_copyable, private std::allocator_traits 0); - link_new_chunk(); } - explicit object_pool(std::size_t chunk_size, const Allocator& allocator) noexcept + // If chunk_size == 0, it is UB + explicit object_pool(size_t chunk_size, const Allocator& allocator) noexcept : byte_allocator(allocator), m_chunk_size{chunk_size}, m_chunk_byte_size{sizeof(chunk_header) + chunk_size * sizeof(obj_slot)} { assert(chunk_size > 0); - link_new_chunk(); } diff --git a/runtime-common/core/memory-resource/segmented-stack-resource.h b/runtime-common/core/memory-resource/segmented-stack-resource.h deleted file mode 100644 index a2a2e81ab4..0000000000 --- a/runtime-common/core/memory-resource/segmented-stack-resource.h +++ /dev/null @@ -1,117 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2026 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#pragma once - -#include -#include - -#include "common/mixin/not_copyable.h" -#include "common/wrappers/likely.h" -#include "runtime-common/core/utils/kphp-assert-core.h" - -namespace memory_resource { - -template -class segmented_stack_resource : private vk::not_copyable { - struct segment_list_node { - std::byte* m_curr{nullptr}; - segment_list_node* m_next{nullptr}; - }; - - std::size_t m_segment_size{0}; - SegmentPool m_segment_pool; - segment_list_node* m_head{nullptr}; - std::byte* m_segment_begin{nullptr}; - std::byte* m_segment_curr{nullptr}; - - auto switch_to_new_segment(void* segment) noexcept -> void { - if (m_head != nullptr) { - m_head->m_curr = m_segment_curr; - } - - m_segment_begin = static_cast(segment) + segment_header_size(); - m_segment_curr = m_segment_begin; - - new (segment) segment_list_node{m_segment_begin, m_head}; - m_head = static_cast(segment); - } - - auto switch_to_old_segment() noexcept -> void { - segment_list_node* top_segment{m_head}; - m_head = m_head->m_next; - m_segment_pool.deallocate(top_segment); - - m_segment_begin = reinterpret_cast(m_head) + segment_header_size(); - m_segment_curr = m_head->m_curr; - } - -public: - auto init(void* buffer, size_t buffer_size, size_t segment_size) noexcept -> void { - php_assert(segment_size > 0); - - m_segment_size = segment_size; - m_segment_pool.init(buffer, buffer_size, m_segment_size + segment_header_size()); - - void* segment{m_segment_pool.allocate()}; - if (unlikely(segment == nullptr)) { - return; - } - - switch_to_new_segment(segment); - } - - auto allocate(size_t size) noexcept -> void* { - if (unlikely(size > m_segment_size)) { - return nullptr; - } - - if (unlikely(m_head == nullptr || size > m_segment_size - static_cast(m_segment_curr - m_segment_begin))) { - void* new_segment{m_segment_pool.allocate()}; - if (unlikely(new_segment == nullptr)) { - return nullptr; - } - - switch_to_new_segment(new_segment); - } - - void* allocated{m_segment_curr}; - m_segment_curr += size; - - return allocated; - } - - auto allocate0(size_t size) noexcept -> void* { - void* allocated{allocate(size)}; - if (likely(allocated != nullptr)) { - memset(allocated, 0x00, size); - } - - return allocated; - } - - auto deallocate(void* mem, size_t size) noexcept -> void { - php_assert(m_segment_curr - static_cast(mem) == size); - - m_segment_curr -= size; - - if (m_segment_curr == m_segment_begin && m_head->m_next != nullptr) { - switch_to_old_segment(); - } - } - - auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { - m_segment_pool.add_extra_memory(buffer, buffer_size); - } - - auto get_buffer_list_head() noexcept -> auto* { - return m_segment_pool.get_buffer_list_head(); - } - - static auto segment_header_size() noexcept -> size_t { - return sizeof(segment_list_node); - } -}; - -} // namespace memory_resource diff --git a/runtime-common/core/memory-resource/stack-resource.h b/runtime-common/core/memory-resource/stack-resource.h index 04d1b12da1..fa139ad24b 100644 --- a/runtime-common/core/memory-resource/stack-resource.h +++ b/runtime-common/core/memory-resource/stack-resource.h @@ -55,6 +55,10 @@ class stack_resource : private vk::not_copyable { auto memory_begin() noexcept -> void* { return m_begin; } + + auto free_memory_size() noexcept -> size_t { + return m_end - m_curr; + } }; } // namespace memory_resource diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index db9f74aabc..d1647ac4c2 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -31,7 +31,7 @@ struct ComponentState final : private vk::not_copyable { static constexpr auto DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE{static_cast(16U * 1024U * 1024U)}; // 16MiB static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE{static_cast(1U * 1024U * 1024U)}; // 1MiB static constexpr auto DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB - static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE = static_cast(32U * 1024U); // 32KiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE = static_cast(32U * 1024U); // 32KiB static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE = static_cast(80U); static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB /** Leaves the same five-percent runway between legacy kPHP's hard OOM threshold and its memory limit. */ @@ -57,7 +57,7 @@ struct ComponentState final : private vk::not_copyable { size_t m_initial_instance_coroutine_memory_size{DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; size_t m_min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; size_t m_initial_instance_task_memory_size{DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE}; - size_t m_instance_task_allocator_segment_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE}; + size_t m_instance_task_allocator_stack_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE}; size_t m_instance_task_allocator_stack_pool_chunk_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE}; size_t m_min_instance_extra_task_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE}; diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 56f738a197..aeb6050ecb 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -65,7 +65,7 @@ struct InstanceState final : vk::not_copyable { m_component_state.m_min_instance_extra_coroutine_memory_size, 0, m_component_state.m_initial_instance_task_memory_size, - m_component_state.m_instance_task_allocator_segment_size, + m_component_state.m_instance_task_allocator_stack_size, m_component_state.m_instance_task_allocator_stack_pool_chunk_size, m_component_state.m_min_instance_extra_task_memory_size, 0}; diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index 6c78c5fc29..c8187145d3 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -159,13 +159,13 @@ void ComponentState::parse_initial_instance_task_memory_size_arg(std::string_vie kphp::log::info("set initial instance task memory size to {} bytes", initial_instance_task_memory_size); } -void ComponentState::parse_instance_task_allocator_segment_size_arg(std::string_view value_view) noexcept { +void ComponentState::parse_instance_task_allocator_stack_size_arg(std::string_view value_view) noexcept { const auto parsed{parse_uint64(value_view)}; if (!parsed) { kphp::log::error("couldn't parse instance task allocator segment size, got {}", value_view); } - instance_task_allocator_segment_size = *parsed; - kphp::log::info("set instance task allocator segment size to {} bytes", instance_task_allocator_segment_size); + instance_task_allocator_stack_size = *parsed; + kphp::log::info("set instance task allocator segment size to {} bytes", instance_task_allocator_stack_size); } void ComponentState::parse_instance_task_allocator_stack_pool_chunk_size_arg(std::string_view value_view) noexcept { @@ -213,7 +213,7 @@ void ComponentState::parse_args() noexcept { } else if (key_view == INITIAL_INSTANCE_TASK_MEMORY_SIZE_ARG) { parse_initial_instance_task_memory_size_arg(value_view); } else if (key_view == INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE_ARG) { - parse_instance_task_allocator_segment_size_arg(value_view); + parse_instance_task_allocator_stack_size_arg(value_view); } else if (key_view == INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE_ARG) { parse_instance_task_allocator_stack_pool_chunk_size_arg(value_view); } else if (key_view == MIN_INSTANCE_EXTRA_TASK_MEMORY_SIZE_ARG) { diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index 97be4f5028..66c2be969d 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -34,7 +34,7 @@ struct ComponentState final : private vk::not_copyable { uint64_t initial_instance_coroutine_memory_size{DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; uint64_t initial_instance_task_memory_size{DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE}; - uint64_t instance_task_allocator_segment_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE}; + uint64_t instance_task_allocator_stack_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE}; uint64_t instance_task_allocator_stack_pool_chunk_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE}; uint64_t min_instance_extra_task_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE}; @@ -82,7 +82,7 @@ struct ComponentState final : private vk::not_copyable { static constexpr auto DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB static constexpr auto DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB - static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE = static_cast(32U * 1024U); // 32KiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE = static_cast(32U * 1024U); // 32KiB static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE = static_cast(80U); static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB @@ -110,7 +110,7 @@ struct ComponentState final : private vk::not_copyable { void parse_initial_instance_task_memory_size_arg(std::string_view) noexcept; - void parse_instance_task_allocator_segment_size_arg(std::string_view) noexcept; + void parse_instance_task_allocator_stack_size_arg(std::string_view) noexcept; void parse_instance_task_allocator_stack_pool_chunk_size_arg(std::string_view) noexcept; diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 4f2e475fd7..4a19f45661 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -96,7 +96,7 @@ struct InstanceState final : vk::not_copyable { component_state.min_instance_extra_coroutine_memory_size, 0, component_state.initial_instance_task_memory_size, - component_state.instance_task_allocator_segment_size, + component_state.instance_task_allocator_stack_size, component_state.instance_task_allocator_stack_pool_chunk_size, component_state.min_instance_extra_task_memory_size, 0}; diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 1749bf42ba..720d6482bc 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -16,10 +16,10 @@ namespace kphp::coro { struct instance_state final : private vk::not_copyable { instance_state(size_t coroutine_mem_size, size_t min_extra_coroutine_mem_size, size_t oom_handling_coroutine_mem_size, size_t task_mem_size, - size_t task_allocator_segment_size, size_t task_allocator_stack_pool_chunk_size, size_t min_extra_task_mem_size, + size_t task_allocator_stack_size, size_t task_allocator_stack_pool_chunk_size, size_t min_extra_task_mem_size, size_t oom_handling_task_mem_size) noexcept : coroutine_allocator{coroutine_mem_size, min_extra_coroutine_mem_size, oom_handling_coroutine_mem_size}, - task_allocator{task_mem_size, task_allocator_segment_size, task_allocator_stack_pool_chunk_size, min_extra_task_mem_size, oom_handling_task_mem_size} {} + task_allocator{task_mem_size, task_allocator_stack_size, task_allocator_stack_pool_chunk_size, min_extra_task_mem_size, oom_handling_task_mem_size} {} static instance_state& get() noexcept; diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h index 374b5ce19e..d95bd60af2 100644 --- a/runtime-light/coroutine/detached-task.h +++ b/runtime-light/coroutine/detached-task.h @@ -132,7 +132,7 @@ struct detached_task { using awaiter_base = detail::awaiter_base; struct awaiter final : public awaiter_base, private kphp::coro::task_allocator_scope { awaiter(std::coroutine_handle coro, kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* stack) noexcept + memory_resource::stack_resource* stack) noexcept : awaiter_base(coro), kphp::coro::task_allocator_scope(task_allocator, stack) {} @@ -166,7 +166,7 @@ struct detached_task { private: std::coroutine_handle m_coro; kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; - memory_resource::segmented_stack_resource* m_stack{nullptr}; + memory_resource::stack_resource* m_stack{nullptr}; template friend auto kphp::coro::detach_task(F&& f, Args&&... args) noexcept; diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h index c30bc762ff..ac71ddf247 100644 --- a/runtime-light/coroutine/detail/allocator/task-allocator.h +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -11,8 +11,7 @@ #include "runtime-common/core/allocator/platform-malloc-interface.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/memory-resource/chunk-pool-resource.h" -#include "runtime-common/core/memory-resource/segmented-stack-resource.h" -#include "runtime-common/core/std/containers.h" +#include "runtime-common/core/memory-resource/stack-resource.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro::detail::memory { @@ -25,43 +24,11 @@ inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept - } // namespace task struct task_allocator final : private vk::not_copyable { - struct shared_chunk_pool { - memory_resource::chunk_pool_resource& m_chunk_pool; - - shared_chunk_pool() noexcept - : m_chunk_pool{task_allocator::get().m_chunk_pool} {} - - auto init(void* /*unused*/, size_t /*unused*/, size_t /*unused*/) noexcept -> void {} - - auto allocate() noexcept -> void* { - return m_chunk_pool.allocate(); - } - - auto allocate0() noexcept -> void* { - return m_chunk_pool.allocate0(); - } - - auto deallocate(void* mem) noexcept -> void { - m_chunk_pool.deallocate(mem); - } - - auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { - m_chunk_pool.add_extra_memory(buffer, buffer_size); - } - - auto get_buffer_list_head() const noexcept -> auto* { - return m_chunk_pool.get_buffer_list_head(); - } - }; - private: - vk::object_pool, - kphp::memory::script_allocator>> - m_stack_pool; + vk::object_pool> m_stack_pool; memory_resource::chunk_pool_resource m_chunk_pool; - kphp::stl::vector*, kphp::memory::script_allocator> m_free_stacks; - memory_resource::segmented_stack_resource* m_curr_stack{nullptr}; - size_t m_segment_size{0}; + memory_resource::stack_resource* m_curr_stack{nullptr}; + size_t m_stack_size{0}; size_t m_min_extra_mem_size{0}; auto request_extra_memory(size_t requested_size) noexcept -> void { @@ -81,23 +48,18 @@ struct task_allocator final : private vk::not_copyable { public: static auto get() noexcept -> task_allocator&; - task_allocator(size_t script_mem_size, size_t segment_size, size_t stack_pool_chunk_size, size_t min_extra_mem_size, size_t /*unused*/) noexcept + task_allocator(size_t script_mem_size, size_t stack_size, size_t stack_pool_chunk_size, size_t min_extra_mem_size, size_t /*unused*/) noexcept : m_stack_pool{stack_pool_chunk_size}, - m_segment_size{segment_size}, + m_stack_size{stack_size}, m_min_extra_mem_size{min_extra_mem_size} { void* buffer{kphp::memory::platform::alloc(script_mem_size)}; kphp::log::assertion(buffer != nullptr); - m_chunk_pool.init(buffer, script_mem_size, m_segment_size + memory_resource::segmented_stack_resource::segment_header_size()); - m_free_stacks.reserve(stack_pool_chunk_size); + m_chunk_pool.init(buffer, script_mem_size, m_stack_size); } auto free() noexcept -> void { - for (auto* stack : m_free_stacks) { - m_stack_pool.release(*stack); - } - auto* curr_buffer{m_chunk_pool.get_buffer_list_head()}; while (curr_buffer != nullptr) { auto* next_buffer = curr_buffer->next; @@ -106,39 +68,15 @@ struct task_allocator final : private vk::not_copyable { } } - auto acquire_stack() noexcept -> memory_resource::segmented_stack_resource& { - if (m_free_stacks.empty()) { - auto& stack{m_stack_pool.acquire()}; - // We can pass nullptr as buffer and 0 as buffer_size, because init in shared_chunk_pool is noop - stack.init(nullptr, 0, m_segment_size); - - return stack; - } - - auto* stack{m_free_stacks.back()}; - m_free_stacks.pop_back(); - - return *stack; - } - - auto release_stack(memory_resource::segmented_stack_resource& stack) noexcept -> void { - m_free_stacks.push_back(std::addressof(stack)); - } - - auto segment_size() const noexcept -> size_t { - return m_segment_size; - } - - auto current_stack() const noexcept -> memory_resource::segmented_stack_resource* { + auto current_stack() const noexcept -> memory_resource::stack_resource* { return m_curr_stack; } - auto set_stack(memory_resource::segmented_stack_resource* stack) noexcept -> void { + auto set_stack(memory_resource::stack_resource* stack) noexcept -> void { m_curr_stack = stack; } - auto exchange_stack(memory_resource::segmented_stack_resource* stack) noexcept - -> memory_resource::segmented_stack_resource* { + auto exchange_stack(memory_resource::stack_resource* stack) noexcept -> memory_resource::stack_resource* { auto* prev{m_curr_stack}; set_stack(stack); @@ -146,16 +84,32 @@ struct task_allocator final : private vk::not_copyable { } private: - auto alloc_script_memory(size_t size) noexcept -> void* { - void* mem{m_curr_stack->allocate(size)}; - if (mem == nullptr) [[unlikely]] { - request_extra_memory(size); - mem = m_curr_stack->allocate(size); + auto stack_size() const noexcept -> size_t { + return m_stack_size; + } + + auto acquire_stack() noexcept -> memory_resource::stack_resource& { + auto& stack{m_stack_pool.acquire()}; + auto* chunk{m_chunk_pool.allocate()}; + if (chunk == nullptr) [[unlikely]] { + request_extra_memory(m_stack_size); + chunk = m_chunk_pool.allocate(); - kphp::log::assertion(mem != nullptr); + kphp::log::assertion(chunk != nullptr); } - return mem; + stack.init(chunk, m_stack_size); + + return stack; + } + + auto release_stack(memory_resource::stack_resource& stack) noexcept -> void { + m_chunk_pool.deallocate(stack.memory_begin()); + m_stack_pool.release(stack); + } + + auto alloc_script_memory(size_t size) noexcept -> void* { + return m_curr_stack->allocate(size); } auto free_script_memory(void* mem, size_t size) noexcept -> void { diff --git a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h index 13eff701c9..4e37526cec 100644 --- a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h +++ b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h @@ -10,6 +10,7 @@ #include #include +#include "runtime-common/core/memory-resource/stack-resource.h" #include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -20,7 +21,7 @@ struct control_block { public: enum class backend_type : uint8_t { stack, stack_owner, coroutine_pool }; - memory_resource::segmented_stack_resource* stack{nullptr}; + memory_resource::stack_resource* stack{nullptr}; uint16_t base_offset{}; backend_type backend{}; @@ -42,7 +43,7 @@ inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { void* base{nullptr}; kphp::coro::detail::memory::task::control_block::backend_type backend{}; auto& task_allocator{kphp::coro::detail::memory::task_allocator::get()}; - if (total_size <= task_allocator.segment_size()) [[likely]] { + if (total_size <= task_allocator.stack_size()) [[likely]] { if (task_allocator.current_stack() == nullptr) { auto& stack{task_allocator.acquire_stack()}; task_allocator.set_stack(std::addressof(stack)); @@ -52,7 +53,9 @@ inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { } base = task_allocator.alloc_script_memory(total_size); - } else { + } + + if (base == nullptr) { base = kphp::coro::detail::memory::runtime_coroutine_allocator::get().alloc_script_memory(total_size); backend = kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool; } @@ -88,8 +91,7 @@ inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept - task_allocator.release_stack(*cb->stack); break; case kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool: - [[unlikely]] kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(reinterpret_cast(ptr) - cb->base_offset, - total_size); + kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); break; } } diff --git a/runtime-light/coroutine/task-allocator-guard.h b/runtime-light/coroutine/task-allocator-guard.h index ce70c519d6..94e3867c50 100644 --- a/runtime-light/coroutine/task-allocator-guard.h +++ b/runtime-light/coroutine/task-allocator-guard.h @@ -5,7 +5,7 @@ #pragma once #include "common/mixin/not_copyable.h" -#include "runtime-common/core/memory-resource/segmented-stack-resource.h" +#include "runtime-common/core/memory-resource/stack-resource.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" namespace kphp::coro { @@ -13,7 +13,7 @@ namespace kphp::coro { class task_allocator_guard : private vk::not_copyable { private: kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; - memory_resource::segmented_stack_resource* m_stack{nullptr}; + memory_resource::stack_resource* m_stack{nullptr}; public: explicit task_allocator_guard() noexcept diff --git a/runtime-light/coroutine/task-allocator-scope.h b/runtime-light/coroutine/task-allocator-scope.h index c3384a2e51..27cdc18f0c 100644 --- a/runtime-light/coroutine/task-allocator-scope.h +++ b/runtime-light/coroutine/task-allocator-scope.h @@ -5,7 +5,7 @@ #pragma once #include "common/mixin/not_copyable.h" -#include "runtime-common/core/memory-resource/segmented-stack-resource.h" +#include "runtime-common/core/memory-resource/stack-resource.h" #include "runtime-light/coroutine/detail/allocator/task-allocator.h" namespace kphp::coro { @@ -13,14 +13,13 @@ namespace kphp::coro { class task_allocator_scope : private vk::not_copyable { private: kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; - memory_resource::segmented_stack_resource* m_stack{nullptr}; + memory_resource::stack_resource* m_stack{nullptr}; public: - explicit task_allocator_scope(memory_resource::segmented_stack_resource* stack) noexcept + explicit task_allocator_scope(memory_resource::stack_resource* stack) noexcept : m_stack{m_task_allocator.exchange_stack(stack)} {} - explicit task_allocator_scope(kphp::coro::detail::memory::task_allocator& task_allocator, - memory_resource::segmented_stack_resource* stack) noexcept + explicit task_allocator_scope(kphp::coro::detail::memory::task_allocator& task_allocator, memory_resource::stack_resource* stack) noexcept : m_task_allocator{task_allocator}, m_stack{m_task_allocator.exchange_stack(stack)} {} diff --git a/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp b/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp deleted file mode 100644 index c5fbad7b89..0000000000 --- a/tests/cpp/runtime/memory_resource/segmented-stack-resource-test.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Compiler for PHP (aka KPHP) -// Copyright (c) 2026 LLC «V Kontakte» -// Distributed under the GPL v3 License, see LICENSE.notice.txt - -#include -#include -#include -#include - -#include - -#include "runtime-common/core/memory-resource/chunk-pool-resource.h" -#include "runtime-common/core/memory-resource/segmented-stack-resource.h" - -namespace { - -constexpr size_t SEGMENT_SIZE{64}; -// mirrors segment_list_node's layout: two pointers, no padding -constexpr size_t SEGMENT_HEADER_SIZE{2 * sizeof(void*)}; -// mirrors chunk_pool_resource's internal buffer_list_node layout: a single pointer, no padding -constexpr size_t POOL_HEADER_SIZE{sizeof(void*)}; -// size of one chunk as seen by the underlying SegmentPool: segment payload + segment_list_node header -constexpr size_t SEGMENT_CHUNK_SIZE{SEGMENT_SIZE + SEGMENT_HEADER_SIZE}; - -// heap-allocated to avoid -Warray-bounds false positives in gcc -[[gnu::noinline]] auto make_buffer(size_t segments) -> std::vector { - return std::vector(POOL_HEADER_SIZE + segments * SEGMENT_CHUNK_SIZE); -} - -auto segment_payload(std::byte* buffer_start, size_t index) noexcept -> std::byte* { - return buffer_start + POOL_HEADER_SIZE + index * SEGMENT_CHUNK_SIZE + SEGMENT_HEADER_SIZE; -} - -} // namespace - -TEST(segmented_stack_resource_test, uninited_state) { - memory_resource::segmented_stack_resource resource; - - ASSERT_EQ(resource.get_buffer_list_head(), nullptr); -} - -TEST(segmented_stack_resource_test, single_allocation_exact_segment_fit) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* mem{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_EQ(mem, static_cast(segment_payload(buffer.data(), 0))); - // the single segment is fully consumed, and there's no more pool memory for a new one - ASSERT_EQ(resource.allocate(1), nullptr); -} - -TEST(segmented_stack_resource_test, sequential_allocations_bump_within_segment) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* a{resource.allocate(16)}; - void* b{resource.allocate(16)}; - void* c{resource.allocate(32)}; - - ASSERT_EQ(a, static_cast(segment_payload(buffer.data(), 0))); - ASSERT_EQ(b, static_cast(segment_payload(buffer.data(), 0) + 16)); - ASSERT_EQ(c, static_cast(segment_payload(buffer.data(), 0) + 32)); - // exactly filled the segment - ASSERT_EQ(resource.allocate(1), nullptr); -} - -TEST(segmented_stack_resource_test, allocation_crossing_segment_boundary_switches_segment) { - auto buffer{make_buffer(2)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* first{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_EQ(first, static_cast(segment_payload(buffer.data(), 0))); - - // doesn't fit in the remaining space of the first segment, must switch to a new one - void* second{resource.allocate(1)}; - - ASSERT_EQ(second, static_cast(segment_payload(buffer.data(), 1))); - - ASSERT_EQ(resource.allocate(SEGMENT_SIZE), nullptr); // pool exhausted -} - -TEST(segmented_stack_resource_test, lifo_deallocate_reuses_address_within_segment) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - resource.allocate(16); - void* b1{resource.allocate(16)}; - - resource.deallocate(b1, 16); - void* b2{resource.allocate(16)}; - - ASSERT_EQ(b2, b1); -} - -TEST(segmented_stack_resource_test, full_drain_returns_segment_to_pool_and_resets_state) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* mem1{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_NE(mem1, nullptr); - - resource.deallocate(mem1, SEGMENT_SIZE); - - // the segment was released back to the pool, so a fresh allocation gets the same address - void* mem2{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_EQ(mem2, mem1); -} - -TEST(segmented_stack_resource_test, regrowth_after_full_drain_across_multiple_segments) { - auto buffer{make_buffer(2)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* first{resource.allocate(SEGMENT_SIZE)}; - void* second{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_NE(first, nullptr); - ASSERT_NE(second, nullptr); - - // unwind in LIFO order - resource.deallocate(second, SEGMENT_SIZE); - resource.deallocate(first, SEGMENT_SIZE); - - void* first_again{resource.allocate(SEGMENT_SIZE)}; - void* second_again{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_EQ(first_again, first); - ASSERT_EQ(second_again, second); -} - -TEST(segmented_stack_resource_test, multi_segment_unwind_restores_saved_cursor) { - auto buffer{make_buffer(2)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* a{resource.allocate(48)}; // segment 0, [0, 48) - void* b{resource.allocate(32)}; // doesn't fit remaining 16 bytes of segment 0, switches to segment 1 - void* c{resource.allocate(16)}; // segment 1, right after b - - ASSERT_EQ(a, static_cast(segment_payload(buffer.data(), 0))); - ASSERT_EQ(b, static_cast(segment_payload(buffer.data(), 1))); - ASSERT_EQ(c, static_cast(segment_payload(buffer.data(), 1) + 32)); - - resource.deallocate(c, 16); - resource.deallocate(b, 32); // drains segment 1 back to the pool - - // segment 0's cursor (saved when we switched away from it) must be restored correctly - void* d{resource.allocate(16)}; - ASSERT_EQ(d, static_cast(segment_payload(buffer.data(), 0) + 48)); -} - -TEST(segmented_stack_resource_test, exhaustion_returns_null_without_corrupting_state) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* mem{resource.allocate(SEGMENT_SIZE - 16)}; - - ASSERT_NE(mem, nullptr); - - // doesn't fit in the remaining 16 bytes, and there's no more pool memory for a new segment - ASSERT_EQ(resource.allocate(32), nullptr); - - // the failed attempt must not have corrupted the cursor of the current segment - void* rest{resource.allocate(16)}; - - ASSERT_EQ(rest, static_cast(segment_payload(buffer.data(), 0) + (SEGMENT_SIZE - 16))); -} - -TEST(segmented_stack_resource_test, allocate0_returns_zeroed_memory) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - void* mem{resource.allocate(SEGMENT_SIZE)}; - - ASSERT_NE(mem, nullptr); - - resource.deallocate(mem, SEGMENT_SIZE); - - void* mem0{resource.allocate0(SEGMENT_SIZE)}; - - ASSERT_EQ(mem0, mem); - - auto* bytes{static_cast(mem0)}; - for (size_t i = 0; i < SEGMENT_SIZE; ++i) { - ASSERT_EQ(static_cast(bytes[i]), 0); - } -} - -TEST(segmented_stack_resource_test, allocate0_on_exhausted_pool_returns_null) { - auto buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); - ASSERT_EQ(resource.allocate0(1), nullptr); -} - -TEST(segmented_stack_resource_test, segments_do_not_overlap) { - constexpr size_t SEGMENTS{4}; - auto buffer{make_buffer(SEGMENTS)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer.data(), buffer.size(), SEGMENT_SIZE); - - std::array segments{}; - for (size_t i = 0; i < SEGMENTS; ++i) { - segments[i] = resource.allocate(SEGMENT_SIZE); - - ASSERT_NE(segments[i], nullptr); - - std::memset(segments[i], static_cast(i + 1), SEGMENT_SIZE); - } - - for (size_t i = 0; i < SEGMENTS; ++i) { - auto* bytes{static_cast(segments[i])}; - for (size_t j = 0; j < SEGMENT_SIZE; ++j) { - ASSERT_EQ(bytes[j], static_cast(i + 1)); - } - } -} - -TEST(segmented_stack_resource_test, add_extra_memory_grows_pool_after_exhaustion) { - auto buffer1{make_buffer(1)}; - auto buffer2{make_buffer(2)}; - memory_resource::segmented_stack_resource resource; - resource.init(buffer1.data(), buffer1.size(), SEGMENT_SIZE); - - ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); - ASSERT_EQ(resource.allocate(1), nullptr); // exhausted - - resource.add_extra_memory(buffer2.data(), buffer2.size()); - - ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); - ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); - ASSERT_EQ(resource.allocate(1), nullptr); // exhausted again -} - -TEST(segmented_stack_resource_test, add_extra_memory_before_exhaustion_preserves_existing_free_segment) { - auto old_buffer{make_buffer(2)}; - auto extra_buffer{make_buffer(1)}; - memory_resource::segmented_stack_resource resource; - resource.init(old_buffer.data(), old_buffer.size(), SEGMENT_SIZE); - - // consume the first segment of old_buffer; the second one is left free in the pool, not yet exhausted - ASSERT_NE(resource.allocate(SEGMENT_SIZE), nullptr); - - void* remaining_old_segment{static_cast(segment_payload(old_buffer.data(), 1))}; - - resource.add_extra_memory(extra_buffer.data(), extra_buffer.size()); - - // the newly added buffer's segment is handed out before old_buffer's still-unused leftover segment - ASSERT_EQ(resource.allocate(SEGMENT_SIZE), static_cast(segment_payload(extra_buffer.data(), 0))); - ASSERT_EQ(resource.allocate(SEGMENT_SIZE), remaining_old_segment); - ASSERT_EQ(resource.allocate(1), nullptr); // exhausted -} - -TEST(segmented_stack_resource_test, buffer_list_head_tracks_most_recently_added_buffer) { - auto buffer1{make_buffer(1)}; - auto buffer2{make_buffer(1)}; - - memory_resource::segmented_stack_resource resource; - resource.init(buffer1.data(), buffer1.size(), SEGMENT_SIZE); - - ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer1.data())); - - resource.add_extra_memory(buffer2.data(), buffer2.size()); - - ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer2.data())); - - auto* head{resource.get_buffer_list_head()}; - - ASSERT_EQ(static_cast(head->next), static_cast(buffer1.data())); - - head = head->next; - - ASSERT_EQ(static_cast(head->next), nullptr); -} diff --git a/tests/cpp/runtime/runtime-tests.cmake b/tests/cpp/runtime/runtime-tests.cmake index bf413fd41a..e1ec8124fd 100644 --- a/tests/cpp/runtime/runtime-tests.cmake +++ b/tests/cpp/runtime/runtime-tests.cmake @@ -19,7 +19,6 @@ prepend(RUNTIME_TESTS_SOURCES ${BASE_DIR}/tests/cpp/runtime/ memory_resource/details/memory_chunk_tree-test.cpp memory_resource/details/memory_ordered_chunk_list-test.cpp memory_resource/extra-memory-pool-test.cpp - memory_resource/segmented-stack-resource-test.cpp memory_resource/stack-resource-test.cpp memory_resource/unsynchronized_pool_resource-test.cpp string-list-test.cpp From 1bf2984ff40803365a83b459a37588569e7eb3bb Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 2 Nov 2026 17:56:36 +0300 Subject: [PATCH 22/26] format --- runtime-light/coroutine/coroutine.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime-light/coroutine/coroutine.cmake b/runtime-light/coroutine/coroutine.cmake index 4d6cfd8a58..8960525ed2 100644 --- a/runtime-light/coroutine/coroutine.cmake +++ b/runtime-light/coroutine/coroutine.cmake @@ -1,3 +1,4 @@ set(RUNTIME_LIGHT_COROUTINE_SRC coroutine/detail/allocator/task-allocator.cpp - coroutine/detail/allocator/runtime-coroutine-allocator.cpp) \ No newline at end of file + coroutine/detail/allocator/runtime-coroutine-allocator.cpp) + \ No newline at end of file From f8195cc3e1a60500e229d324d4b5201c7a98fa59 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 2 Nov 2026 17:58:20 +0300 Subject: [PATCH 23/26] format --- runtime-light/coroutine/coroutine.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime-light/coroutine/coroutine.cmake b/runtime-light/coroutine/coroutine.cmake index 8960525ed2..ec20c24b41 100644 --- a/runtime-light/coroutine/coroutine.cmake +++ b/runtime-light/coroutine/coroutine.cmake @@ -1,4 +1,3 @@ set(RUNTIME_LIGHT_COROUTINE_SRC coroutine/detail/allocator/task-allocator.cpp coroutine/detail/allocator/runtime-coroutine-allocator.cpp) - \ No newline at end of file From 6c40fa5430f4ae90d9f0994e7b3ec0144b972743 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 2 Nov 2026 18:03:55 +0300 Subject: [PATCH 24/26] format --- runtime-common/stdlib/array/array-functions.h | 4 ++-- .../stdlib/component/inter-component-session/client.h | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/runtime-common/stdlib/array/array-functions.h b/runtime-common/stdlib/array/array-functions.h index 127df98d65..d16f79e1e4 100644 --- a/runtime-common/stdlib/array/array-functions.h +++ b/runtime-common/stdlib/array/array-functions.h @@ -1160,8 +1160,8 @@ inline Optional> f$array_column(const array& a, const mixed& } template -auto f$array_column(const Optional& a, const mixed& column_key, - const mixed& index_key = {}) noexcept -> decltype(f$array_column(std::declval(), column_key, index_key)) { +auto f$array_column(const Optional& a, const mixed& column_key, const mixed& index_key = {}) noexcept + -> decltype(f$array_column(std::declval(), column_key, index_key)) { if (!a.has_value()) { php_warning("first parameter of array_column must be array"); return false; diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 3f86f1744d..c17e3e84cb 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -141,7 +141,7 @@ class client final { reader(const shared_ctx_type& _ctx, shared_transport_type _t) noexcept : ctx(_ctx), interrupter(reader::wait_until_interrupt(_ctx)), - runner(reader::run(_ctx, std::move(_t), interrupter)){}; + runner(reader::run(_ctx, std::move(_t), interrupter)) {} static auto run(shared_ctx_type ctx, shared_transport_type t, kphp::coro::shared_task interrupter) noexcept -> kphp::coro::shared_task { // Allocate buffer for header @@ -281,7 +281,7 @@ class client final { template B, std::invocable> R> requires std::is_convertible_v, std::span> && - std::same_as>, client::response_readiness> + std::same_as>, client::response_readiness> auto query(std::span request, B response_buffer_provider, R response_handler) noexcept -> kphp::coro::task>; template B> @@ -300,9 +300,9 @@ inline auto client::create(std::string_view component_name) noexcept -> std::exp template B, std::invocable> R> requires std::is_convertible_v, std::span> && - std::same_as>, client::response_readiness> -auto client::query(std::span request, B response_buffer_provider, - R response_handler) noexcept -> kphp::coro::task> { + std::same_as>, client::response_readiness> +auto client::query(std::span request, B response_buffer_provider, R response_handler) noexcept + -> kphp::coro::task> { // If any reader error has previously occurred if (reader.ctx.get() == nullptr) [[unlikely]] { co_return std::unexpected(k2::errno_eshutdown); From 53da8d10a891e678bd6e01dd4daa2255d865f9e9 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Mon, 2 Nov 2026 18:10:43 +0300 Subject: [PATCH 25/26] format --- runtime-common/stdlib/array/array-functions.h | 4 ++-- .../stdlib/component/inter-component-session/client.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime-common/stdlib/array/array-functions.h b/runtime-common/stdlib/array/array-functions.h index d16f79e1e4..127df98d65 100644 --- a/runtime-common/stdlib/array/array-functions.h +++ b/runtime-common/stdlib/array/array-functions.h @@ -1160,8 +1160,8 @@ inline Optional> f$array_column(const array& a, const mixed& } template -auto f$array_column(const Optional& a, const mixed& column_key, const mixed& index_key = {}) noexcept - -> decltype(f$array_column(std::declval(), column_key, index_key)) { +auto f$array_column(const Optional& a, const mixed& column_key, + const mixed& index_key = {}) noexcept -> decltype(f$array_column(std::declval(), column_key, index_key)) { if (!a.has_value()) { php_warning("first parameter of array_column must be array"); return false; diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index c17e3e84cb..3283b0104b 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -281,7 +281,7 @@ class client final { template B, std::invocable> R> requires std::is_convertible_v, std::span> && - std::same_as>, client::response_readiness> + std::same_as>, client::response_readiness> auto query(std::span request, B response_buffer_provider, R response_handler) noexcept -> kphp::coro::task>; template B> @@ -300,9 +300,9 @@ inline auto client::create(std::string_view component_name) noexcept -> std::exp template B, std::invocable> R> requires std::is_convertible_v, std::span> && - std::same_as>, client::response_readiness> -auto client::query(std::span request, B response_buffer_provider, R response_handler) noexcept - -> kphp::coro::task> { + std::same_as>, client::response_readiness> +auto client::query(std::span request, B response_buffer_provider, + R response_handler) noexcept -> kphp::coro::task> { // If any reader error has previously occurred if (reader.ctx.get() == nullptr) [[unlikely]] { co_return std::unexpected(k2::errno_eshutdown); From 9e67bc42cd50dd8f47240e8bf1fde059e637c364 Mon Sep 17 00:00:00 2001 From: Kirill Kotliar Date: Tue, 3 Nov 2026 13:06:45 +0300 Subject: [PATCH 26/26] add zero sized chunk handling in object pool --- common/containers/object-pool-test.cpp | 97 +++++++++++++------ common/containers/object-pool.h | 28 ++++-- .../components/kphp/state/component-state.cpp | 3 + .../detail/allocator/task-allocator.h | 13 ++- .../detail/allocator/task-malloc-interface.h | 5 +- 5 files changed, 99 insertions(+), 47 deletions(-) diff --git a/common/containers/object-pool-test.cpp b/common/containers/object-pool-test.cpp index 66b12ad181..55ad3f6a95 100644 --- a/common/containers/object-pool-test.cpp +++ b/common/containers/object-pool-test.cpp @@ -89,9 +89,9 @@ struct dtor_tracker { TEST(object_pool_test, acquire_forward_no_constructor_arguments) { vk::object_pool pool{4}; - int& x{pool.acquire()}; + int* x{pool.acquire()}; - ASSERT_EQ(x, 0); + ASSERT_EQ(*x, 0); pool.release(x); } @@ -99,9 +99,9 @@ TEST(object_pool_test, acquire_forward_no_constructor_arguments) { TEST(object_pool_test, acquire_forwards_single_constructor_argument) { vk::object_pool pool{4}; - int& x{pool.acquire(42)}; + int* x{pool.acquire(42)}; - ASSERT_EQ(x, 42); + ASSERT_EQ(*x, 42); pool.release(x); } @@ -109,10 +109,10 @@ TEST(object_pool_test, acquire_forwards_single_constructor_argument) { TEST(object_pool_test, acquire_forwards_multiple_constructor_arguments) { vk::object_pool pool{4}; - point& p{pool.acquire(1, 2)}; + point* p{pool.acquire(1, 2)}; - ASSERT_EQ(p.x, 1); - ASSERT_EQ(p.y, 2); + ASSERT_EQ(p->x, 1); + ASSERT_EQ(p->y, 2); pool.release(p); } @@ -120,10 +120,10 @@ TEST(object_pool_test, acquire_forwards_multiple_constructor_arguments) { TEST(object_pool_test, acquire_forwards_move_only_arguments) { vk::object_pool> pool{4}; - std::unique_ptr& obj{pool.acquire(std::make_unique(5))}; + std::unique_ptr* obj{pool.acquire(std::make_unique(5))}; ASSERT_NE(obj, nullptr); - ASSERT_EQ(*obj, 5); + ASSERT_EQ(**obj, 5); pool.release(obj); } @@ -132,7 +132,7 @@ TEST(object_pool_test, release_invokes_destructor) { vk::object_pool pool{4}; bool destroyed{false}; - dtor_tracker& obj{pool.acquire(&destroyed)}; + dtor_tracker* obj{pool.acquire(&destroyed)}; ASSERT_FALSE(destroyed); @@ -144,13 +144,13 @@ TEST(object_pool_test, release_invokes_destructor) { TEST(object_pool_test, acquire_after_release_reuses_last_freed_slot) { vk::object_pool pool{4}; - int& a{pool.acquire(1)}; - int* addr_a{std::addressof(a)}; + int* a{pool.acquire(1)}; + int* addr_a{a}; pool.release(a); - int& b{pool.acquire(2)}; + int* b{pool.acquire(2)}; - ASSERT_EQ(std::addressof(b), addr_a); + ASSERT_EQ(b, addr_a); pool.release(b); } @@ -163,7 +163,7 @@ TEST(object_pool_test, constructor_allocates_exactly_one_chunk) { ASSERT_EQ(counting_allocator::allocate_calls, 1); - int& x{pool.acquire(7)}; + int* x{pool.acquire(7)}; pool.release(x); } @@ -175,7 +175,7 @@ TEST(object_pool_test, constructor_with_allocator_allocates_exactly_one_chunk) { ASSERT_EQ(counting_allocator::allocate_calls, 1); - int& x{pool.acquire(7)}; + int* x{pool.acquire(7)}; pool.release(x); } @@ -186,7 +186,7 @@ TEST(object_pool_test, constructor_with_allocator_stores_the_passed_allocator_in std::vector ptrs; for (int i = 0; i < 5; ++i) { - ptrs.push_back(std::addressof(pool.acquire(i))); + ptrs.push_back(pool.acquire(i)); } ASSERT_FALSE(tagged_allocator::allocation_tags.empty()); @@ -195,7 +195,7 @@ TEST(object_pool_test, constructor_with_allocator_stores_the_passed_allocator_in } for (auto* p : ptrs) { - pool.release(*p); + pool.release(p); } } @@ -206,13 +206,13 @@ TEST(object_pool_test, acquiring_exactly_chunk_size_objects_does_not_grow) { vk::object_pool> pool{4}; std::vector ptrs; for (int i = 0; i < 4; ++i) { - ptrs.push_back(std::addressof(pool.acquire(i))); + ptrs.push_back(pool.acquire(i)); } ASSERT_EQ(counting_allocator::allocate_calls, 1); for (auto* p : ptrs) { - pool.release(*p); + pool.release(p); } } @@ -221,13 +221,13 @@ TEST(object_pool_test, release_then_acquire_does_not_allocate_a_new_chunk) { counting_allocator::deallocate_calls = 0; vk::object_pool> pool{2}; - int& a{pool.acquire(1)}; - int& b{pool.acquire(2)}; + int* a{pool.acquire(1)}; + int* b{pool.acquire(2)}; ASSERT_EQ(counting_allocator::allocate_calls, 1); pool.release(a); - int& c{pool.acquire(3)}; + int* c{pool.acquire(3)}; ASSERT_EQ(counting_allocator::allocate_calls, 1); @@ -242,13 +242,13 @@ TEST(object_pool_test, allocates_chunks_when_exchausted) { vk::object_pool> pool{2}; std::vector ptrs; for (int i = 0; i < 5; ++i) { - ptrs.push_back(std::addressof(pool.acquire(i))); + ptrs.push_back(pool.acquire(i)); } ASSERT_EQ(counting_allocator::allocate_calls, 3); for (auto* p : ptrs) { - pool.release(*p); + pool.release(p); } } @@ -260,10 +260,10 @@ TEST(object_pool_test, destructor_deallocates_every_allocated_chunk) { vk::object_pool> pool{2}; std::vector ptrs; for (int i = 0; i < 5; ++i) { - ptrs.push_back(std::addressof(pool.acquire(i))); + ptrs.push_back(pool.acquire(i)); } for (auto* p : ptrs) { - pool.release(*p); + pool.release(p); } } @@ -277,7 +277,7 @@ TEST(object_pool_test, acquires_returns_distinct_objects) { std::vector ptrs; for (int i = 0; i < total; ++i) { - ptrs.push_back(std::addressof(pool.acquire(i))); + ptrs.push_back(pool.acquire(i)); } std::set unique_ptrs(ptrs.begin(), ptrs.end()); @@ -288,6 +288,45 @@ TEST(object_pool_test, acquires_returns_distinct_objects) { } for (auto* p : ptrs) { - pool.release(*p); + pool.release(p); } } + +TEST(object_pool_test, zero_chunk_size_does_not_allocate) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + { + vk::object_pool> pool{0}; + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + ASSERT_EQ(counting_allocator::deallocate_calls, 0); +} + +TEST(object_pool_test, zero_chunk_size_with_allocator_does_not_allocate) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + { + vk::object_pool> pool{0, counting_allocator{}}; + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + ASSERT_EQ(counting_allocator::deallocate_calls, 0); +} + +TEST(object_pool_test, zero_chunk_size_acquire_returns_nullptr) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{0}; + + ASSERT_EQ(pool.acquire(), nullptr); + ASSERT_EQ(pool.acquire(42), nullptr); + ASSERT_EQ(counting_allocator::allocate_calls, 0); +} diff --git a/common/containers/object-pool.h b/common/containers/object-pool.h index 3e0a89389c..d83fa0d968 100644 --- a/common/containers/object-pool.h +++ b/common/containers/object-pool.h @@ -55,20 +55,24 @@ class object_pool : private vk::not_copyable, private std::allocator_traits 0); + if (unlikely(m_chunk_size == 0)) { + return; + } + link_new_chunk(); } - // If chunk_size == 0, it is UB explicit object_pool(size_t chunk_size, const Allocator& allocator) noexcept : byte_allocator(allocator), m_chunk_size{chunk_size}, m_chunk_byte_size{sizeof(chunk_header) + chunk_size * sizeof(obj_slot)} { - assert(chunk_size > 0); + if (unlikely(m_chunk_size == 0)) { + return; + } + link_new_chunk(); } @@ -82,9 +86,13 @@ class object_pool : private vk::not_copyable, private std::allocator_traits - auto acquire(Args&&... args) noexcept -> T& { + auto acquire(Args&&... args) noexcept -> T* { + if (unlikely(m_chunk_size == 0)) { + return nullptr; + } + if (unlikely(m_head_free_slot == nullptr)) { link_new_chunk(); } @@ -94,13 +102,13 @@ class object_pool : private vk::not_copyable, private std::allocator_traitsm_obj), std::forward(args)...); - return free_slot->m_obj; + return std::addressof(free_slot->m_obj); } - auto release(T& obj) noexcept -> void { - byte_allocator_traits::destroy(get_byte_allocator_ref(), std::addressof(obj)); + auto release(T* obj) noexcept -> void { + byte_allocator_traits::destroy(get_byte_allocator_ref(), obj); - auto* slot{reinterpret_cast(std::addressof(obj))}; + auto* slot{reinterpret_cast(obj)}; slot->m_next = m_head_free_slot; m_head_free_slot = slot; } diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index c8187145d3..d0292faf91 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -173,6 +173,9 @@ void ComponentState::parse_instance_task_allocator_stack_pool_chunk_size_arg(std if (!parsed) { kphp::log::error("couldn't parse instance task allocator stack pool chunk size, got {}", value_view); } + if (*parsed == 0) { + kphp::log::error("instance task allocator stack pool chunk size must be greater than 0"); + } instance_task_allocator_stack_pool_chunk_size = *parsed; kphp::log::info("set instance task allocator stack pool chunk size to {} objects", instance_task_allocator_stack_pool_chunk_size); } diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h index ac71ddf247..06285a714f 100644 --- a/runtime-light/coroutine/detail/allocator/task-allocator.h +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -52,6 +52,9 @@ struct task_allocator final : private vk::not_copyable { : m_stack_pool{stack_pool_chunk_size}, m_stack_size{stack_size}, m_min_extra_mem_size{min_extra_mem_size} { + // `acquire_stack` relies on the pool never returning nullptr + kphp::log::assertion(stack_pool_chunk_size > 0); + void* buffer{kphp::memory::platform::alloc(script_mem_size)}; kphp::log::assertion(buffer != nullptr); @@ -88,8 +91,8 @@ struct task_allocator final : private vk::not_copyable { return m_stack_size; } - auto acquire_stack() noexcept -> memory_resource::stack_resource& { - auto& stack{m_stack_pool.acquire()}; + auto acquire_stack() noexcept -> memory_resource::stack_resource* { + auto* stack{m_stack_pool.acquire()}; auto* chunk{m_chunk_pool.allocate()}; if (chunk == nullptr) [[unlikely]] { request_extra_memory(m_stack_size); @@ -98,13 +101,13 @@ struct task_allocator final : private vk::not_copyable { kphp::log::assertion(chunk != nullptr); } - stack.init(chunk, m_stack_size); + stack->init(chunk, m_stack_size); return stack; } - auto release_stack(memory_resource::stack_resource& stack) noexcept -> void { - m_chunk_pool.deallocate(stack.memory_begin()); + auto release_stack(memory_resource::stack_resource* stack) noexcept -> void { + m_chunk_pool.deallocate(stack->memory_begin()); m_stack_pool.release(stack); } diff --git a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h index 4e37526cec..d835c4c515 100644 --- a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h +++ b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h @@ -45,8 +45,7 @@ inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { auto& task_allocator{kphp::coro::detail::memory::task_allocator::get()}; if (total_size <= task_allocator.stack_size()) [[likely]] { if (task_allocator.current_stack() == nullptr) { - auto& stack{task_allocator.acquire_stack()}; - task_allocator.set_stack(std::addressof(stack)); + task_allocator.set_stack(task_allocator.acquire_stack()); backend = kphp::coro::detail::memory::task::control_block::backend_type::stack_owner; } else { backend = kphp::coro::detail::memory::task::control_block::backend_type::stack; @@ -88,7 +87,7 @@ inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept - task_allocator.free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); task_allocator.set_stack(nullptr); - task_allocator.release_stack(*cb->stack); + task_allocator.release_stack(cb->stack); break; case kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool: kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size);