diff --git a/common/common-tests.cmake b/common/common-tests.cmake index 5fd0b42b23..1a0a35f7e8 100644 --- a/common/common-tests.cmake +++ b/common/common-tests.cmake @@ -18,7 +18,8 @@ prepend(COMMON_TESTS_SOURCES ${COMMON_DIR}/ wrappers/span-test.cpp wrappers/string_view-test.cpp ucontext/ucontext-portable-test.cpp - containers/intrusive-list-test.cpp) + containers/intrusive-list-test.cpp + containers/object-pool-test.cpp) allow_deprecated_declarations(${COMMON_TESTS_SOURCES}/algorithms/projections-test.cpp) diff --git a/common/containers/object-pool-test.cpp b/common/containers/object-pool-test.cpp new file mode 100644 index 0000000000..55ad3f6a95 --- /dev/null +++ b/common/containers/object-pool-test.cpp @@ -0,0 +1,332 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include +#include + +#include + +#include "common/containers/object-pool.h" + +namespace { + +template +struct counting_allocator : std::allocator { + using std::allocator::allocator; + + template + struct rebind { + using other = counting_allocator; + }; + + static inline size_t allocate_calls{0}; + static inline size_t deallocate_calls{0}; + + auto allocate(std::size_t n) -> T* { + ++allocate_calls; + return std::allocator::allocate(n); + } + + auto deallocate(T* p, std::size_t n) -> void { + ++deallocate_calls; + std::allocator::deallocate(p, n); + } +}; + +template +struct tagged_allocator : std::allocator { + using std::allocator::allocator; + + template + struct rebind { + using other = tagged_allocator; + }; + + static inline std::vector allocation_tags; + + int m_tag{0}; + + tagged_allocator() = default; + explicit tagged_allocator(int tag) noexcept + : m_tag{tag} {} + + template + tagged_allocator(const tagged_allocator& other) noexcept + : m_tag{other.m_tag} {} + + auto allocate(std::size_t n) -> T* { + allocation_tags.push_back(m_tag); + return std::allocator::allocate(n); + } +}; + +struct point { + int x{}; + int y{}; + + point() = default; + point(int x_val, int y_val) noexcept + : x{x_val}, + y{y_val} {} +}; + +struct dtor_tracker { + bool* m_destroyed{nullptr}; + + explicit dtor_tracker(bool* destroyed) noexcept + : m_destroyed{destroyed} {} + + ~dtor_tracker() { + *m_destroyed = true; + } +}; + +} // namespace + +TEST(object_pool_test, acquire_forward_no_constructor_arguments) { + vk::object_pool pool{4}; + + int* x{pool.acquire()}; + + ASSERT_EQ(*x, 0); + + pool.release(x); +} + +TEST(object_pool_test, acquire_forwards_single_constructor_argument) { + vk::object_pool pool{4}; + + int* x{pool.acquire(42)}; + + ASSERT_EQ(*x, 42); + + pool.release(x); +} + +TEST(object_pool_test, acquire_forwards_multiple_constructor_arguments) { + vk::object_pool pool{4}; + + point* p{pool.acquire(1, 2)}; + + ASSERT_EQ(p->x, 1); + ASSERT_EQ(p->y, 2); + + pool.release(p); +} + +TEST(object_pool_test, acquire_forwards_move_only_arguments) { + vk::object_pool> pool{4}; + + std::unique_ptr* obj{pool.acquire(std::make_unique(5))}; + + ASSERT_NE(obj, nullptr); + ASSERT_EQ(**obj, 5); + + pool.release(obj); +} + +TEST(object_pool_test, release_invokes_destructor) { + vk::object_pool pool{4}; + bool destroyed{false}; + + dtor_tracker* obj{pool.acquire(&destroyed)}; + + ASSERT_FALSE(destroyed); + + pool.release(obj); + + ASSERT_TRUE(destroyed); +} + +TEST(object_pool_test, acquire_after_release_reuses_last_freed_slot) { + vk::object_pool pool{4}; + + int* a{pool.acquire(1)}; + int* addr_a{a}; + pool.release(a); + + int* b{pool.acquire(2)}; + + ASSERT_EQ(b, addr_a); + + pool.release(b); +} + +TEST(object_pool_test, constructor_allocates_exactly_one_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{3}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + int* x{pool.acquire(7)}; + pool.release(x); +} + +TEST(object_pool_test, constructor_with_allocator_allocates_exactly_one_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{3, counting_allocator{}}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + int* x{pool.acquire(7)}; + pool.release(x); +} + +TEST(object_pool_test, constructor_with_allocator_stores_the_passed_allocator_instance) { + tagged_allocator::allocation_tags.clear(); + + vk::object_pool> pool{2, tagged_allocator{42}}; + + std::vector ptrs; + for (int i = 0; i < 5; ++i) { + ptrs.push_back(pool.acquire(i)); + } + + ASSERT_FALSE(tagged_allocator::allocation_tags.empty()); + for (int tag : tagged_allocator::allocation_tags) { + ASSERT_EQ(tag, 42); + } + + for (auto* p : ptrs) { + pool.release(p); + } +} + +TEST(object_pool_test, acquiring_exactly_chunk_size_objects_does_not_grow) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{4}; + std::vector ptrs; + for (int i = 0; i < 4; ++i) { + ptrs.push_back(pool.acquire(i)); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + for (auto* p : ptrs) { + pool.release(p); + } +} + +TEST(object_pool_test, release_then_acquire_does_not_allocate_a_new_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{2}; + int* a{pool.acquire(1)}; + int* b{pool.acquire(2)}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + pool.release(a); + int* c{pool.acquire(3)}; + + ASSERT_EQ(counting_allocator::allocate_calls, 1); + + pool.release(b); + pool.release(c); +} + +TEST(object_pool_test, allocates_chunks_when_exchausted) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{2}; + std::vector ptrs; + for (int i = 0; i < 5; ++i) { + ptrs.push_back(pool.acquire(i)); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 3); + + for (auto* p : ptrs) { + pool.release(p); + } +} + +TEST(object_pool_test, destructor_deallocates_every_allocated_chunk) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + { + vk::object_pool> pool{2}; + std::vector ptrs; + for (int i = 0; i < 5; ++i) { + ptrs.push_back(pool.acquire(i)); + } + for (auto* p : ptrs) { + pool.release(p); + } + } + + ASSERT_EQ(counting_allocator::allocate_calls, counting_allocator::deallocate_calls); + ASSERT_GE(counting_allocator::allocate_calls, 3); +} + +TEST(object_pool_test, acquires_returns_distinct_objects) { + constexpr int total{10}; + vk::object_pool pool{2}; + std::vector ptrs; + + for (int i = 0; i < total; ++i) { + ptrs.push_back(pool.acquire(i)); + } + + std::set unique_ptrs(ptrs.begin(), ptrs.end()); + ASSERT_EQ(unique_ptrs.size(), total); + + for (int i = 0; i < total; ++i) { + ASSERT_EQ(*ptrs[i], i); + } + + for (auto* p : ptrs) { + pool.release(p); + } +} + +TEST(object_pool_test, zero_chunk_size_does_not_allocate) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + { + vk::object_pool> pool{0}; + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + ASSERT_EQ(counting_allocator::deallocate_calls, 0); +} + +TEST(object_pool_test, zero_chunk_size_with_allocator_does_not_allocate) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + { + vk::object_pool> pool{0, counting_allocator{}}; + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + } + + ASSERT_EQ(counting_allocator::allocate_calls, 0); + ASSERT_EQ(counting_allocator::deallocate_calls, 0); +} + +TEST(object_pool_test, zero_chunk_size_acquire_returns_nullptr) { + counting_allocator::allocate_calls = 0; + counting_allocator::deallocate_calls = 0; + + vk::object_pool> pool{0}; + + ASSERT_EQ(pool.acquire(), nullptr); + ASSERT_EQ(pool.acquire(42), nullptr); + ASSERT_EQ(counting_allocator::allocate_calls, 0); +} diff --git a/common/containers/object-pool.h b/common/containers/object-pool.h new file mode 100644 index 0000000000..d83fa0d968 --- /dev/null +++ b/common/containers/object-pool.h @@ -0,0 +1,117 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +#include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" + +namespace vk { + +template> +class object_pool : private vk::not_copyable, private std::allocator_traits::template rebind_alloc { + using byte_allocator = typename std::allocator_traits::template rebind_alloc; + using byte_allocator_traits = std::allocator_traits; + + union obj_slot { + obj_slot* m_next; + T m_obj; + }; + + struct alignas(alignof(obj_slot)) chunk_header { + chunk_header* m_next{nullptr}; + }; + + size_t m_chunk_size{0}; + size_t m_chunk_byte_size{0}; + chunk_header* m_head_chunk{nullptr}; + obj_slot* m_head_free_slot{nullptr}; + + auto get_byte_allocator_ref() noexcept -> byte_allocator& { + return *this; + } + + static auto slots_of(chunk_header* chunk) noexcept -> obj_slot* { + return reinterpret_cast(reinterpret_cast(chunk) + sizeof(chunk_header)); + } + + auto link_new_chunk() noexcept -> void { + std::byte* mem{byte_allocator_traits::allocate(get_byte_allocator_ref(), m_chunk_byte_size)}; + m_head_chunk = new (mem) chunk_header{m_head_chunk}; + + obj_slot* slots{slots_of(m_head_chunk)}; + slots[0].m_next = nullptr; + for (size_t i = 1; i < m_chunk_size; ++i) { + slots[i].m_next = std::addressof(slots[i - 1]); + } + + m_head_free_slot = std::addressof(slots[m_chunk_size - 1]); + } + +public: + explicit object_pool(size_t chunk_size) noexcept + : m_chunk_size{chunk_size}, + m_chunk_byte_size{sizeof(chunk_header) + chunk_size * sizeof(obj_slot)} { + if (unlikely(m_chunk_size == 0)) { + return; + } + + link_new_chunk(); + } + + explicit object_pool(size_t chunk_size, const Allocator& allocator) noexcept + : byte_allocator(allocator), + m_chunk_size{chunk_size}, + m_chunk_byte_size{sizeof(chunk_header) + chunk_size * sizeof(obj_slot)} { + if (unlikely(m_chunk_size == 0)) { + return; + } + + link_new_chunk(); + } + + // All acquired objects must be released before destructor call + ~object_pool() { + auto* curr_chunk{m_head_chunk}; + while (curr_chunk != nullptr) { + auto* next_chunk{curr_chunk->m_next}; + byte_allocator_traits::deallocate(get_byte_allocator_ref(), reinterpret_cast(curr_chunk), m_chunk_byte_size); + curr_chunk = next_chunk; + } + } + + // Returns nullptr only if size of chunk is 0 + template + auto acquire(Args&&... args) noexcept -> T* { + if (unlikely(m_chunk_size == 0)) { + return nullptr; + } + + if (unlikely(m_head_free_slot == nullptr)) { + link_new_chunk(); + } + + obj_slot* free_slot{m_head_free_slot}; + m_head_free_slot = m_head_free_slot->m_next; + + byte_allocator_traits::construct(get_byte_allocator_ref(), std::addressof(free_slot->m_obj), std::forward(args)...); + + return std::addressof(free_slot->m_obj); + } + + auto release(T* obj) noexcept -> void { + byte_allocator_traits::destroy(get_byte_allocator_ref(), obj); + + auto* slot{reinterpret_cast(obj)}; + slot->m_next = m_head_free_slot; + m_head_free_slot = slot; + } +}; + +} // namespace vk diff --git a/compiler/code-gen/files/init-scripts.cpp b/compiler/code-gen/files/init-scripts.cpp index 57b3988697..60e0ee3255 100644 --- a/compiler/code-gen/files/init-scripts.cpp +++ b/compiler/code-gen/files/init-scripts.cpp @@ -259,7 +259,7 @@ void InitScriptsCpp::compile(CodeGenerator& W) const { if (G->is_output_mode_k2()) { FunctionSignatureGenerator(W) << "void init_php_scripts_in_each_worker(" << PhpMutableGlobalsRefArgument() - << ", kphp::coro::task<> &run" + << ", kphp::coro::detached_task<> &run" ")" << BEGIN; } else { @@ -276,7 +276,7 @@ void InitScriptsCpp::compile(CodeGenerator& W) const { W << FunctionName(main_file_id->main_function) << "$globals_reset(php_globals);" << NL; if (G->is_output_mode_k2()) { - W << "run = " << FunctionName(main_file_id->main_function) << "$run();" << NL; + W << "run = kphp::coro::detach_task(" << FunctionName(main_file_id->main_function) << "$run);" << NL; } else { W << "set_script (" << FunctionName(main_file_id->main_function) << "$run, " << FunctionName(main_file_id->main_function) << "$globals_reset);" << NL; } @@ -320,11 +320,7 @@ void ComponentInfoFile::compile(CodeGenerator& W) const { << R"(static std::array extraInfo {ImageInfo::KeyValuePair{.key = "compiler_version", .value = ")" << G->settings().get_version() << "\"}};" << NL << "static ImageInfo imageInfo {\"" << G->settings().k2_component_name.get() << "\"" << "," << (G->is_output_mode_k2_multishot() ? "0" : "1") << "," << std::to_string(std::chrono::duration_cast(G->settings().build_tp.time_since_epoch()).count()) << "," - << "K2_PLATFORM_HEADER_H_VERSION, " - << "{" << php_code_commit_hash << "}," - << "\"" << G->settings().php_code_version.get() << "\"," - << "extraInfo.size()" << "," - << "extraInfo.data()" - << "};" << NL << "return &imageInfo;" << NL << END; + << "K2_PLATFORM_HEADER_H_VERSION, " << "{" << php_code_commit_hash << "}," << "\"" << G->settings().php_code_version.get() << "\"," << "extraInfo.size()" + << "," << "extraInfo.data()" << "};" << NL << "return &imageInfo;" << NL << END; W << CloseFile(); } diff --git a/compiler/code-gen/vertex-compiler.cpp b/compiler/code-gen/vertex-compiler.cpp index ee9bd862a3..93583db2c9 100644 --- a/compiler/code-gen/vertex-compiler.cpp +++ b/compiler/code-gen/vertex-compiler.cpp @@ -932,7 +932,7 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ if (mode == func_call_mode::fork_call) { if (func->is_interruptible) { - W << "(kphp::forks::start(" << FunctionName(func); + W << "(kphp::forks::start(kphp::coro::detach_task([](auto&&... args) noexcept { return " << FunctionName(func); } else { W << FunctionForkName(func); } @@ -947,7 +947,12 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ const TypeData* tp = tinf::get_type(root); W << "< " << TypeName(tp) << " >"; } - W << "("; + + if (func->is_interruptible && mode == func_call_mode::fork_call) { + W << "(std::forward(args)...); }"; + } else { + W << "("; + } if (func && func->is_extern() && vk::any_of_equal(func->name, "JsonEncoder$$to_json_impl", "JsonEncoder$$from_json_impl")) { root = patch_compiling_json_impl_call(W, root); @@ -962,6 +967,10 @@ void compile_func_call(VertexAdaptor root, CodeGenerator& W, func_ } } + if (!args.empty() && func->is_interruptible && mode == func_call_mode::fork_call) { + W << ", "; + } + W << JoinValues(args, ", "); if (is_function_call_should_be_tracked(func)) { W << "))"; @@ -2428,8 +2437,7 @@ void compile_common_op(VertexPtr root, CodeGenerator& W) { } case op_arr_acc_set_return: { auto v = root.as(); - W << "ARR_ACC_SET_RETURN" << MacroBegin{} << v->obj() << ", " << v->offset() << ", " << v->value() << ", " - << "f$" << v->set_method->name << MacroEnd{}; + W << "ARR_ACC_SET_RETURN" << MacroBegin{} << v->obj() << ", " << v->offset() << ", " << v->value() << ", " << "f$" << v->set_method->name << MacroEnd{}; break; } case op_arr_acc_check_and_get: { @@ -2441,9 +2449,7 @@ void compile_common_op(VertexPtr root, CodeGenerator& W) { W << "ARR_ACC_GET_IF_ISSET"; } - W << MacroBegin{} << v->obj() << ", " << v->offset() << ", " - << "f$" << v->check_method->name << ", " - << "f$" << v->get_method->name << MacroEnd{}; + W << MacroBegin{} << v->obj() << ", " << v->offset() << ", " << "f$" << v->check_method->name << ", " << "f$" << v->get_method->name << MacroEnd{}; break; } default: diff --git a/runtime-common/core/memory-resource/chunk-pool-resource.h b/runtime-common/core/memory-resource/chunk-pool-resource.h new file mode 100644 index 0000000000..20faccff08 --- /dev/null +++ b/runtime-common/core/memory-resource/chunk-pool-resource.h @@ -0,0 +1,106 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include + +#include "common/mixin/not_copyable.h" +#include "common/wrappers/likely.h" +#include "runtime-common/core/memory-resource/memory_resource.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace memory_resource { + +class chunk_pool_resource : private vk::not_copyable { + struct buffer_list_node { + buffer_list_node* next{nullptr}; + }; + + struct chunk_free_list_node { + chunk_free_list_node* next{nullptr}; + }; + + std::size_t m_chunk_size{0}; + chunk_free_list_node* m_head_chunk{nullptr}; + buffer_list_node* m_head_buffer{nullptr}; + + auto link_new_buffer(void* buffer, size_t buffer_size) noexcept -> void { + new (buffer) buffer_list_node{m_head_buffer}; + + std::byte* first_chunk{static_cast(buffer) + buffer_header_size()}; + + size_t payload_size{buffer_size - buffer_header_size()}; + size_t used_buffer_size{payload_size - payload_size % m_chunk_size}; + + std::byte* curr_chunk{first_chunk}; + std::byte* last_chunk{curr_chunk + used_buffer_size - m_chunk_size}; + while (curr_chunk != last_chunk) { + std::byte* next_chunk = curr_chunk + m_chunk_size; + new (curr_chunk) chunk_free_list_node{reinterpret_cast(next_chunk)}; + curr_chunk = next_chunk; + } + + new (last_chunk) chunk_free_list_node{m_head_chunk}; + + m_head_chunk = reinterpret_cast(first_chunk); + } + +public: + auto init(void* buffer, size_t buffer_size, size_t chunk_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= chunk_size + buffer_header_size() && chunk_size >= chunk_header_size() && + chunk_size % alignof(chunk_free_list_node) == 0); + + m_chunk_size = chunk_size; + link_new_buffer(buffer, buffer_size); + m_head_buffer = static_cast(buffer); + } + + auto allocate() noexcept -> void* { + if (unlikely(m_head_chunk == nullptr)) { + return nullptr; + } + + void* allocated_chunk{m_head_chunk}; + m_head_chunk = m_head_chunk->next; + + return allocated_chunk; + } + + auto allocate0() noexcept -> void* { + void* allocated_chunk{allocate()}; + if (likely(allocated_chunk != nullptr)) { + memset(allocated_chunk, 0x00, m_chunk_size); + } + + return allocated_chunk; + } + + auto deallocate(void* mem) noexcept -> void { + m_head_chunk = new (mem) chunk_free_list_node{m_head_chunk}; + } + + auto add_extra_memory(void* buffer, size_t buffer_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit() && buffer_size >= m_chunk_size + buffer_header_size()); + + link_new_buffer(buffer, buffer_size); + m_head_buffer = static_cast(buffer); + } + + auto get_buffer_list_head() const noexcept -> buffer_list_node* { + return m_head_buffer; + } + + static auto chunk_header_size() noexcept -> std::size_t { + return sizeof(chunk_free_list_node); + } + + static auto buffer_header_size() noexcept -> std::size_t { + return sizeof(buffer_list_node); + } +}; + +} // namespace memory_resource diff --git a/runtime-common/core/memory-resource/stack-resource.h b/runtime-common/core/memory-resource/stack-resource.h new file mode 100644 index 0000000000..fa139ad24b --- /dev/null +++ b/runtime-common/core/memory-resource/stack-resource.h @@ -0,0 +1,64 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/memory_resource.h" +#include "runtime-common/core/utils/kphp-assert-core.h" + +namespace memory_resource { + +class stack_resource : private vk::not_copyable { + std::byte* m_begin{nullptr}; + std::byte* m_curr{nullptr}; + std::byte* m_end{nullptr}; + +public: + auto init(void* buffer, size_t buffer_size) noexcept -> void { + php_assert(buffer_size <= memory_buffer_limit()); + + m_begin = static_cast(buffer); + m_curr = m_begin; + m_end = m_curr + buffer_size; + } + + auto allocate(size_t size) noexcept -> void* { + if (size > static_cast(m_end - m_curr)) { + return nullptr; + } + + void* allocated{m_curr}; + m_curr += size; + + return allocated; + } + + auto allocate0(size_t size) noexcept -> void* { + void* allocated{allocate(size)}; + if (allocated != nullptr) { + memset(allocated, 0x00, size); + } + + return allocated; + } + + auto deallocate(void* mem, size_t size) noexcept -> void { + php_assert(m_curr - static_cast(mem) == size); + m_curr -= size; + } + + auto memory_begin() noexcept -> void* { + return m_begin; + } + + auto free_memory_size() noexcept -> size_t { + return m_end - m_curr; + } +}; + +} // namespace memory_resource diff --git a/runtime-light/components/confdata/confdata.cmake b/runtime-light/components/confdata/confdata.cmake index 2b955c879c..cb579a7215 100644 --- a/runtime-light/components/confdata/confdata.cmake +++ b/runtime-light/components/confdata/confdata.cmake @@ -9,6 +9,9 @@ set(KPHP_CONFDATA_COMPONENT_SRC ${RUNTIME_LIGHT_DIR}/stdlib/confdata/confdata-keys.cpp ${RUNTIME_LIGHT_DIR}/stdlib/confdata/predefined-wildcards.cpp) +set(KPHP_CONFDATA_COROUTINE_SRC ${RUNTIME_LIGHT_COROUTINE_SRC}) +list(TRANSFORM KPHP_CONFDATA_COROUTINE_SRC PREPEND "${RUNTIME_LIGHT_DIR}/") + set(KPHP_CONFDATA_TL_SRC ${RUNTIME_LIGHT_DIR}/tl/tl-types.cpp ${RUNTIME_LIGHT_DIR}/tl/tl-functions.cpp) @@ -32,6 +35,7 @@ list(TRANSFORM KPHP_CONFDATA_RUNTIME_CORE_SRC PREPEND "${RUNTIME_COMMON_DIR}/") set(KPHP_CONFDATA_SRC ${KPHP_CONFDATA_COMPONENT_SRC} + ${KPHP_CONFDATA_COROUTINE_SRC} ${KPHP_CONFDATA_TL_SRC} ${KPHP_CONFDATA_ALLOCATOR_SRC} ${KPHP_CONFDATA_DIAGNOSTICS_SRC} diff --git a/runtime-light/components/confdata/state/component-state.h b/runtime-light/components/confdata/state/component-state.h index 69355c8c7d..d1647ac4c2 100644 --- a/runtime-light/components/confdata/state/component-state.h +++ b/runtime-light/components/confdata/state/component-state.h @@ -24,15 +24,21 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view PREDEFINED_WILDCARDS_ARG{"predefined-wildcards"}; static constexpr std::string_view INITIAL_INSTANCE_MEMORY_SIZE_ARG{"initial-instance-memory-size"}; static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG{"min-instance-extra-memory-size"}; - static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE{static_cast(1024U * 1024U)}; // 1MiB - static constexpr auto DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE{static_cast(64U * 1024U * 1024U)}; // 64MiB - static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_SIZE{64U * 1024U * 1024U}; // 64MiB + static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE{static_cast(1024U * 1024U)}; // 1MiB + static constexpr auto MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE{static_cast(64U * 1024U * 1024U)}; // 64MiB + static constexpr auto DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE{static_cast(16U * 1024U * 1024U)}; // 16MiB + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE{static_cast(1U * 1024U * 1024U)}; // 1MiB + static constexpr auto DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE = static_cast(32U * 1024U); // 32KiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE = static_cast(80U); + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB /** Leaves the same five-percent runway between legacy kPHP's hard OOM threshold and its memory limit. */ static constexpr size_t DEFAULT_OOM_HANDLING_SIZE_DIVISOR{20}; public: - AllocatorState m_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState m_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; private: const uint32_t m_argc{k2::args_count()}; @@ -47,7 +53,13 @@ struct ComponentState final : private vk::not_copyable { kphp::stl::string m_confdata_proxy_actor_name; kphp::stl::vector m_predefined_wildcards; size_t m_initial_instance_memory_size{DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE}; - size_t m_min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_SIZE}; + size_t m_min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE}; + size_t m_initial_instance_coroutine_memory_size{DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + size_t m_min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; + size_t m_initial_instance_task_memory_size{DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE}; + size_t m_instance_task_allocator_stack_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE}; + size_t m_instance_task_allocator_stack_pool_chunk_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE}; + size_t m_min_instance_extra_task_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE}; // === METHODS ================================================================================== ComponentState() noexcept; diff --git a/runtime-light/components/confdata/state/instance-state.cpp b/runtime-light/components/confdata/state/instance-state.cpp index d43c7999e8..92b9488271 100644 --- a/runtime-light/components/confdata/state/instance-state.cpp +++ b/runtime-light/components/confdata/state/instance-state.cpp @@ -25,6 +25,7 @@ #include "runtime-light/components/confdata/confdata-proxy/sync-functions.h" #include "runtime-light/components/confdata/confdata-proxy/tl.h" #include "runtime-light/components/confdata/state/component-state.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/coroutine/when-all.h" @@ -230,7 +231,7 @@ auto InstanceState::erase_if_retired_and_unused(confdata_piece_list::iterator pi } auto InstanceState::init() noexcept -> void { - auto main_task{run()}; + auto main_task{kphp::coro::detach_task(&InstanceState::run, this)}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; main_task_async_stack_frame.async_stack_root = std::addressof(m_coroutine_instance_state.coroutine_stack_root); @@ -240,7 +241,8 @@ auto InstanceState::init() noexcept -> void { } auto InstanceState::run() noexcept -> kphp::coro::task<> { - co_await kphp::coro::when_all(service_loop(), accept_loop(), metrics_loop()); // all never return + co_await kphp::coro::when_all(kphp::coro::detach_task(&InstanceState::service_loop, this), kphp::coro::detach_task(&InstanceState::accept_loop, this), + kphp::coro::detach_task(&InstanceState::metrics_loop, this)); // all never return kphp::log::assertion(false); } @@ -337,7 +339,7 @@ auto InstanceState::accept_loop() noexcept -> kphp::coro::task<> { } kphp::log::debug("accepted a stream: descriptor -> {}", stream->descriptor()); - if (!m_io_scheduler.start(serve_reader_lease(std::move(*stream)))) [[unlikely]] { + if (!m_io_scheduler.start(kphp::coro::detach_task(&InstanceState::serve_reader_lease, this, std::move(*stream)))) [[unlikely]] { kphp::log::warning("failed to serve a confdata reader lease"); } } diff --git a/runtime-light/components/confdata/state/instance-state.h b/runtime-light/components/confdata/state/instance-state.h index 85c09b9248..aeb6050ecb 100644 --- a/runtime-light/components/confdata/state/instance-state.h +++ b/runtime-light/components/confdata/state/instance-state.h @@ -39,10 +39,6 @@ struct InstanceState final : vk::not_copyable { class reader_session; - // === MEMBERS ================================================================================== - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE{static_cast(16U * 1024U * 1024U)}; // 16MiB - static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE{static_cast(1U * 1024U * 1024U)}; // 1MiB - const ComponentState& m_component_state{ComponentState::get()}; public: @@ -65,7 +61,14 @@ struct InstanceState final : vk::not_copyable { std::optional m_update_failure_old_offset; // We only report the last old offset public: - kphp::coro::instance_state m_coroutine_instance_state{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE, DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE, 0}; + kphp::coro::instance_state m_coroutine_instance_state{m_component_state.m_initial_instance_coroutine_memory_size, + m_component_state.m_min_instance_extra_coroutine_memory_size, + 0, + m_component_state.m_initial_instance_task_memory_size, + m_component_state.m_instance_task_allocator_stack_size, + m_component_state.m_instance_task_allocator_stack_pool_chunk_size, + m_component_state.m_min_instance_extra_task_memory_size, + 0}; kphp::coro::io_scheduler m_io_scheduler{m_coroutine_instance_state}; StringInstanceState m_string_state; diff --git a/runtime-light/components/kphp/state/component-state.cpp b/runtime-light/components/kphp/state/component-state.cpp index 251f73d35d..d0292faf91 100644 --- a/runtime-light/components/kphp/state/component-state.cpp +++ b/runtime-light/components/kphp/state/component-state.cpp @@ -150,6 +150,45 @@ void ComponentState::parse_min_instance_extra_coroutine_memory_size_arg(std::str kphp::log::info("set min instance extra coroutine memory size to {} bytes", min_instance_extra_coroutine_memory_size); } +void ComponentState::parse_initial_instance_task_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse initial instance task memory size, got {}", value_view); + } + initial_instance_task_memory_size = *parsed; + kphp::log::info("set initial instance task memory size to {} bytes", initial_instance_task_memory_size); +} + +void ComponentState::parse_instance_task_allocator_stack_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse instance task allocator segment size, got {}", value_view); + } + instance_task_allocator_stack_size = *parsed; + kphp::log::info("set instance task allocator segment size to {} bytes", instance_task_allocator_stack_size); +} + +void ComponentState::parse_instance_task_allocator_stack_pool_chunk_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse instance task allocator stack pool chunk size, got {}", value_view); + } + if (*parsed == 0) { + kphp::log::error("instance task allocator stack pool chunk size must be greater than 0"); + } + instance_task_allocator_stack_pool_chunk_size = *parsed; + kphp::log::info("set instance task allocator stack pool chunk size to {} objects", instance_task_allocator_stack_pool_chunk_size); +} + +void ComponentState::parse_min_instance_extra_task_memory_size_arg(std::string_view value_view) noexcept { + const auto parsed{parse_uint64(value_view)}; + if (!parsed) { + kphp::log::error("couldn't parse min instance extra task memory size, got {}", value_view); + } + min_instance_extra_task_memory_size = *parsed; + kphp::log::info("set min instance extra task memory size to {} bytes", min_instance_extra_task_memory_size); +} + void ComponentState::parse_args() noexcept { for (auto i = 0; i < argc; ++i) { const auto [arg_key, arg_value]{k2::arg_fetch(i)}; @@ -174,6 +213,14 @@ void ComponentState::parse_args() noexcept { parse_initial_instance_coroutine_memory_size_arg(value_view); } else if (key_view == MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG) { parse_min_instance_extra_coroutine_memory_size_arg(value_view); + } else if (key_view == INITIAL_INSTANCE_TASK_MEMORY_SIZE_ARG) { + parse_initial_instance_task_memory_size_arg(value_view); + } else if (key_view == INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE_ARG) { + parse_instance_task_allocator_stack_size_arg(value_view); + } else if (key_view == INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE_ARG) { + parse_instance_task_allocator_stack_pool_chunk_size_arg(value_view); + } else if (key_view == MIN_INSTANCE_EXTRA_TASK_MEMORY_SIZE_ARG) { + parse_min_instance_extra_task_memory_size_arg(value_view); } else { kphp::log::warning("unexpected argument format: {}", key_view); } diff --git a/runtime-light/components/kphp/state/component-state.h b/runtime-light/components/kphp/state/component-state.h index f737cd5c62..66c2be969d 100644 --- a/runtime-light/components/kphp/state/component-state.h +++ b/runtime-light/components/kphp/state/component-state.h @@ -19,7 +19,7 @@ #include "runtime-light/stdlib/kml/kml-state.h" struct ComponentState final : private vk::not_copyable { - AllocatorState component_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; + AllocatorState component_allocator_state{INIT_COMPONENT_ALLOCATOR_SIZE, MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE, 0}; KmlComponentState kml_component_state; // This member does not hold any KPHP types, so setting a reference counter is unnecessary. const uint32_t argc{k2::args_count()}; @@ -29,10 +29,14 @@ struct ComponentState final : private vk::not_copyable { mixed runtime_config; string cluster_name{DEFAULT_CLUSTER_NAME.data(), DEFAULT_CLUSTER_NAME.size()}; bool exit_after_response{}; - uint64_t initial_instance_memory_size{INIT_INSTANCE_ALLOCATOR_SIZE}; + uint64_t initial_instance_memory_size{DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE}; - uint64_t initial_instance_coroutine_memory_size{INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; + uint64_t initial_instance_coroutine_memory_size{DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE}; uint64_t min_instance_extra_coroutine_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE}; + uint64_t initial_instance_task_memory_size{DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE}; + uint64_t instance_task_allocator_stack_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE}; + uint64_t instance_task_allocator_stack_pool_chunk_size{DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE}; + uint64_t min_instance_extra_task_memory_size{DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE}; ComponentState() noexcept { parse_env(); @@ -67,12 +71,20 @@ struct ComponentState final : private vk::not_copyable { static constexpr std::string_view MIN_INSTANCE_EXTRA_MEMORY_SIZE_ARG = "min-instance-extra-memory-size"; static constexpr std::string_view INITIAL_INSTANCE_COROUTINE_MEMORY_SIZE_ARG = "initial-instance-coroutine-memory-size"; static constexpr std::string_view MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_SIZE_ARG = "min-instance-extra-coroutine-memory-size"; + static constexpr std::string_view INITIAL_INSTANCE_TASK_MEMORY_SIZE_ARG = "initial-instance-task-memory-size"; + static constexpr std::string_view INSTANCE_TASK_ALLOCATOR_SEGMENT_SIZE_ARG = "instance-task-allocator-segment-size"; + static constexpr std::string_view INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE_ARG = "instance-task-allocator-stack-pool-chunk-size"; + static constexpr std::string_view MIN_INSTANCE_EXTRA_TASK_MEMORY_SIZE_ARG = "min-instance-extra-task-memory-size"; static constexpr auto INIT_COMPONENT_ALLOCATOR_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB - static constexpr auto DEFAULT_MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto MIN_COMPONENT_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB + static constexpr auto DEFAULT_INIT_INSTANCE_ALLOCATOR_SIZE = static_cast(64U * 1024U * 1024U); // 64MiB static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_MEMORY_POOL_SIZE = static_cast(1024U * 1024U); // 1MiB - static constexpr auto INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB + static constexpr auto DEFAULT_INIT_INSTANCE_COROUTINE_ALLOCATOR_SIZE = static_cast(8U * 1024U * 1024U); // 8MiB static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_COROUTINE_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB + static constexpr auto DEFAULT_INIT_INSTANCE_TASK_ALLOCATOR_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_SIZE = static_cast(32U * 1024U); // 32KiB + static constexpr auto DEFAULT_INSTANCE_TASK_ALLOCATOR_STACK_POOL_CHUNK_SIZE = static_cast(80U); + static constexpr auto DEFAULT_MIN_INSTANCE_EXTRA_TASK_MEMORY_POOL_SIZE = static_cast(4U * 1024U * 1024U); // 4MiB void parse_env() noexcept; @@ -95,4 +107,12 @@ struct ComponentState final : private vk::not_copyable { void parse_initial_instance_coroutine_memory_size_arg(std::string_view) noexcept; void parse_min_instance_extra_coroutine_memory_size_arg(std::string_view) noexcept; + + void parse_initial_instance_task_memory_size_arg(std::string_view) noexcept; + + void parse_instance_task_allocator_stack_size_arg(std::string_view) noexcept; + + void parse_instance_task_allocator_stack_pool_chunk_size_arg(std::string_view) noexcept; + + void parse_min_instance_extra_task_memory_size_arg(std::string_view) noexcept; }; diff --git a/runtime-light/components/kphp/state/instance-state.cpp b/runtime-light/components/kphp/state/instance-state.cpp index 0698f22329..6eebf8e615 100644 --- a/runtime-light/components/kphp/state/instance-state.cpp +++ b/runtime-light/components/kphp/state/instance-state.cpp @@ -59,14 +59,14 @@ consteval std::string_view resolve_sapi_name() noexcept { void InstanceState::init_script_execution() noexcept { runtime_context.init(); - kphp::coro::task<> script_task; + kphp::coro::detached_task<> script_task; init_php_scripts_in_each_worker(php_script_mutable_globals_singleton, script_task); - auto main_task{std::invoke( - [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { + auto main_task{kphp::coro::detach_task( + [](kphp::coro::detached_task<> script_task) noexcept -> kphp::coro::task<> { // wrap script with additional check for unhandled exception - script_task = std::invoke( - [](kphp::coro::task<> script_task) noexcept -> kphp::coro::task<> { + script_task = kphp::coro::detach_task( + [](kphp::coro::detached_task<> script_task) noexcept -> kphp::coro::task<> { co_await script_task; if (auto exception{std::move(ForkInstanceState::get().current_info().get().thrown_exception)}; !exception.is_null()) [[unlikely]] { kphp::log::error("unhandled exception {}", std::move(exception)); diff --git a/runtime-light/components/kphp/state/instance-state.h b/runtime-light/components/kphp/state/instance-state.h index 960cf6b113..4a19f45661 100644 --- a/runtime-light/components/kphp/state/instance-state.h +++ b/runtime-light/components/kphp/state/instance-state.h @@ -93,7 +93,13 @@ struct InstanceState final : vk::not_copyable { kphp::log::contextual_tags instance_tags; kphp::coro::instance_state coroutine_instance_state{component_state.initial_instance_coroutine_memory_size, - component_state.min_instance_extra_coroutine_memory_size, 0}; + component_state.min_instance_extra_coroutine_memory_size, + 0, + component_state.initial_instance_task_memory_size, + component_state.instance_task_allocator_stack_size, + component_state.instance_task_allocator_stack_pool_chunk_size, + component_state.min_instance_extra_task_memory_size, + 0}; kphp::coro::io_scheduler io_scheduler{coroutine_instance_state}; ForkInstanceState fork_instance_state; WaitQueueInstanceState wait_queue_instance_state; @@ -123,7 +129,7 @@ struct InstanceState final : vk::not_copyable { ErrorHandlingState error_handling_instance_state; KmlInstanceState kml_instance_state; - kphp::stl::list, kphp::memory::script_allocator> shutdown_functions; + kphp::stl::list, kphp::memory::script_allocator> shutdown_functions; private: kphp::coro::task<> init_cli_instance() noexcept; diff --git a/runtime-light/core/globals/php-init-scripts.h b/runtime-light/core/globals/php-init-scripts.h index 6c6f3a0046..78515cb168 100644 --- a/runtime-light/core/globals/php-init-scripts.h +++ b/runtime-light/core/globals/php-init-scripts.h @@ -4,11 +4,11 @@ #pragma once -#include "runtime-light/coroutine/task.h" +#include "runtime-light/coroutine/detached-task.h" class PhpScriptMutableGlobals; /// Initializes const variables represented as globals C++ symbols. Definition is generated by compiler. void init_php_scripts_once_in_master() noexcept; /// Initializes mutable globals. Definition is generated by compiler. -void init_php_scripts_in_each_worker(PhpScriptMutableGlobals& php_globals, kphp::coro::task<>& run) noexcept; +void init_php_scripts_in_each_worker(PhpScriptMutableGlobals& php_globals, kphp::coro::detached_task<>& run) noexcept; diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 85ab351a5f..d9b842c492 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -4,9 +4,6 @@ #pragma once -#include -#include - /** * This header defines the data structures used to represent a coroutine asynchronous stack. * @@ -83,17 +80,6 @@ struct async_stack_root { stack_frame* stop_sync_stack_frame{}; }; -/** - * The `resume` function is responsible for storing the current synchronous stack frame - * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows - * capturing one of the stack frames in the synchronous stack trace. - */ -inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root) noexcept { - auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; - handle.resume(); - stack_root.stop_sync_stack_frame = previous_stack_frame; -} - /** * The async_stack_element class facilitates working with asynchronous traces in templated code. * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. diff --git a/runtime-light/coroutine/await-set.h b/runtime-light/coroutine/await-set.h index 3117d46cc7..1f4592bb94 100644 --- a/runtime-light/coroutine/await-set.h +++ b/runtime-light/coroutine/await-set.h @@ -46,7 +46,8 @@ class await_set { await_set& operator=(const await_set&) = delete; template - requires kphp::coro::concepts::awaitable && std::is_same_v::awaiter_return_type, return_type> + requires kphp::coro::concepts::detached_awaitable && + std::is_same_v::awaiter_return_type, return_type> void push(awaitable_type awaitable) noexcept { kphp::log::assertion(m_await_broker != nullptr); m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable)), m_coroutine_stack_root, STACK_RETURN_ADDRESS); diff --git a/runtime-light/coroutine/concepts.h b/runtime-light/coroutine/concepts.h index aa3b507b24..d5c6a3847c 100644 --- a/runtime-light/coroutine/concepts.h +++ b/runtime-light/coroutine/concepts.h @@ -9,6 +9,8 @@ #include #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/detail/type-traits-utils.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" @@ -44,6 +46,9 @@ concept global_co_await_awaitable = requires(T&& t) { template concept awaitable = detail::member_co_await_awaitable || detail::global_co_await_awaitable || detail::awaiter; +template +concept detached_awaitable = awaitable && !is_task_v; + namespace detail { template @@ -60,10 +65,13 @@ auto get_awaiter(awaitable_type&& value) noexcept { } // namespace detail template -concept coroutine = awaitable && (requires { - { static_cast>(std::declval()) }; +concept coroutine = awaitable && (is_task_v || requires { + { static_cast>(std::declval()) }; } || requires { { static_cast>(std::declval()) }; }); +template +concept detached_coroutine = coroutine && !is_task_v; + } // namespace kphp::coro::concepts diff --git a/runtime-light/coroutine/control-functions.h b/runtime-light/coroutine/control-functions.h new file mode 100644 index 0000000000..26cb9ce96b --- /dev/null +++ b/runtime-light/coroutine/control-functions.h @@ -0,0 +1,38 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-scope.h" + +namespace kphp::coro { + +/** + * The `resume` function is responsible for storing the current synchronous stack frame + * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows + * capturing one of the stack frames in the synchronous stack trace. + */ +inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { + kphp::coro::task_allocator_scope scope{task_allocator, nullptr}; + auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; + handle.resume(); + stack_root.stop_sync_stack_frame = previous_stack_frame; +} + +inline void resume(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { + kphp::coro::task_allocator_scope scope{task_allocator, nullptr}; + handle.resume(); +} + +inline void destroy(std::coroutine_handle<> handle, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept { + kphp::coro::task_allocator_scope scope{task_allocator, nullptr}; + handle.destroy(); +} + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index 887fb7b2a4..720d6482bc 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -8,23 +8,23 @@ #include "common/mixin/not_copyable.h" -#include "runtime-common/core/allocator/pool-allocator.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" namespace kphp::coro { struct instance_state final : private vk::not_copyable { - - instance_state(size_t coroutine_mem_size, size_t min_extra_coroutine_mem_size, size_t oom_handling_coroutine_mem_size) noexcept - : coroutine_allocator{coroutine_mem_size, min_extra_coroutine_mem_size, oom_handling_coroutine_mem_size} {} + instance_state(size_t coroutine_mem_size, size_t min_extra_coroutine_mem_size, size_t oom_handling_coroutine_mem_size, size_t task_mem_size, + size_t task_allocator_stack_size, size_t task_allocator_stack_pool_chunk_size, size_t min_extra_task_mem_size, + size_t oom_handling_task_mem_size) noexcept + : coroutine_allocator{coroutine_mem_size, min_extra_coroutine_mem_size, oom_handling_coroutine_mem_size}, + task_allocator{task_mem_size, task_allocator_stack_size, task_allocator_stack_pool_chunk_size, min_extra_task_mem_size, oom_handling_task_mem_size} {} static instance_state& get() noexcept; - static kphp::memory::pool_allocator& get_coroutine_allocator() noexcept { - return kphp::coro::instance_state::get().coroutine_allocator; - } - - kphp::memory::pool_allocator coroutine_allocator; + kphp::coro::detail::memory::runtime_coroutine_allocator coroutine_allocator; + kphp::coro::detail::memory::task_allocator task_allocator; kphp::coro::async_stack_root coroutine_stack_root; }; diff --git a/runtime-light/coroutine/coroutine.cmake b/runtime-light/coroutine/coroutine.cmake new file mode 100644 index 0000000000..ec20c24b41 --- /dev/null +++ b/runtime-light/coroutine/coroutine.cmake @@ -0,0 +1,3 @@ +set(RUNTIME_LIGHT_COROUTINE_SRC + coroutine/detail/allocator/task-allocator.cpp + coroutine/detail/allocator/runtime-coroutine-allocator.cpp) diff --git a/runtime-light/coroutine/detached-task.h b/runtime-light/coroutine/detached-task.h new file mode 100644 index 0000000000..d95bd60af2 --- /dev/null +++ b/runtime-light/coroutine/detached-task.h @@ -0,0 +1,175 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include + +#include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-scope.h" +#include "runtime-light/coroutine/task.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::coro { + +namespace detail { + +template +requires std::invocable +class async_function_return_type; + +template +class awaiter_base { + void push_async_stack_frame(async_stack_frame& caller_frame, void* return_address) noexcept { + async_stack_frame& callee_frame{m_coro.promise().get_async_stack_frame()}; + callee_frame.caller_async_stack_frame = std::addressof(caller_frame); + callee_frame.return_address = return_address; + + auto* async_stack_root{caller_frame.async_stack_root}; + kphp::log::assertion(async_stack_root != nullptr); + callee_frame.async_stack_root = async_stack_root; + async_stack_root->top_async_stack_frame = std::addressof(callee_frame); + } + + void detach_from_async_stack() noexcept { + async_stack_frame& callee_frame{m_coro.promise().get_async_stack_frame()}; + callee_frame.caller_async_stack_frame = nullptr; + } + +protected: + bool m_started{}; + bool m_suspended{}; + std::coroutine_handle m_coro{}; + +public: + explicit awaiter_base(std::coroutine_handle coro) noexcept + : m_coro(coro) {} + + awaiter_base(awaiter_base&& other) noexcept + : m_started(other.m_started), + m_suspended(std::exchange(other.m_suspended, false)), + m_coro(std::exchange(other.m_coro, {})) {} + + awaiter_base(const awaiter_base& other) = delete; + awaiter_base& operator=(const awaiter_base& other) = delete; + awaiter_base& operator=(awaiter_base&& other) = delete; + + ~awaiter_base() { + if (m_coro != nullptr && m_suspended) { + m_coro.promise().m_next = nullptr; + detach_from_async_stack(); + } + } + + auto await_ready() noexcept -> bool { + kphp::log::assertion(!std::exchange(m_started, true)); // to make sure it's not co_awaited more than once + return false; + } + + template caller_promise_type> + [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> std::coroutine_handle { + push_async_stack_frame(awaiting_coroutine.promise().get_async_stack_frame(), STACK_RETURN_ADDRESS); + m_coro.promise().m_next = awaiting_coroutine.address(); + m_suspended = true; + return m_coro; + } + + auto await_resume() noexcept -> void { + m_suspended = false; + } +}; + +} // namespace detail + +template +auto detach_task(F&& f, Args&&... args) noexcept { + auto scope{kphp::coro::task_allocator_scope{nullptr}}; + return kphp::coro::detached_task::type>{ + std::invoke(std::forward(f), std::forward(args)...)}; +} + +template +struct detached_task { +private: + using task_promise_type = kphp::coro::task::promise_type; + + explicit detached_task(kphp::coro::task&& task) noexcept + : m_coro(std::exchange(task.m_coro, {})), + m_stack(m_task_allocator.current_stack()) {} + +public: + detached_task() noexcept = default; + + detached_task(const detached_task& other) noexcept = delete; + + detached_task(detached_task&& other) noexcept + : m_coro(std::exchange(other.m_coro, {})), + m_stack(std::exchange(other.m_stack, nullptr)) {} + + detached_task& operator=(const detached_task& other) noexcept = delete; + + detached_task& operator=(detached_task&& other) noexcept { + std::swap(m_coro, other.m_coro); + std::swap(m_stack, other.m_stack); + + return *this; + } + + ~detached_task() { + if (m_coro) { + kphp::coro::task_allocator_scope scope{m_task_allocator, m_stack}; + m_coro.destroy(); + } + } + + constexpr auto operator co_await() noexcept { + using awaiter_base = detail::awaiter_base; + struct awaiter final : public awaiter_base, private kphp::coro::task_allocator_scope { + awaiter(std::coroutine_handle coro, kphp::coro::detail::memory::task_allocator& task_allocator, + memory_resource::stack_resource* stack) noexcept + : awaiter_base(coro), + kphp::coro::task_allocator_scope(task_allocator, stack) {} + + auto await_resume() noexcept -> T { + awaiter_base::await_resume(); + return awaiter_base::m_coro.promise().result(); + } + }; + + return awaiter{m_coro, m_task_allocator, m_stack}; + } + + auto get_handle() noexcept -> std::coroutine_handle { + return m_coro; + } + + // conversion functions + // + // erase type + explicit operator detached_task<>() && noexcept { + return detached_task<>{std::exchange(m_coro, {})}; + } + + // restore erased type + template + requires(std::same_as) + explicit operator detached_task() && noexcept { + return detached_task{std::exchange(m_coro, {})}; + } + +private: + std::coroutine_handle m_coro; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + memory_resource::stack_resource* m_stack{nullptr}; + + template + friend auto kphp::coro::detach_task(F&& f, Args&&... args) noexcept; +}; + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/detail/allocator/coroutine-allocator.h b/runtime-light/coroutine/detail/allocator/coroutine-allocator.h index 48c495d81f..8246e83e6f 100644 --- a/runtime-light/coroutine/detail/allocator/coroutine-allocator.h +++ b/runtime-light/coroutine/detail/allocator/coroutine-allocator.h @@ -6,7 +6,7 @@ #include -#include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" namespace kphp::coro::detail::memory { @@ -20,11 +20,11 @@ struct coroutine_allocator { explicit coroutine_allocator(const coroutine_allocator& /*unused*/) noexcept {} constexpr value_type* allocate(size_t n) noexcept { - return static_cast(kphp::coro::instance_state::get_coroutine_allocator().alloc_script_memory(n * sizeof(T))); + return static_cast(kphp::coro::detail::memory::runtime_coroutine_allocator::get().alloc_script_memory(n * sizeof(T))); } constexpr void deallocate(T* p, size_t n) noexcept { - kphp::coro::instance_state::get_coroutine_allocator().free_script_memory(p, n * sizeof(T)); + kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(p, n * sizeof(T)); } }; diff --git a/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h b/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h index 6ad337c955..fc25dd7099 100644 --- a/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h +++ b/runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h @@ -8,32 +8,32 @@ #include #include "runtime-common/core/allocator/details/malloc-interface.h" -#include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" namespace kphp::coro::detail::memory { inline auto alloc(size_t size) noexcept -> void* { - return kphp::memory::details::malloc_interface::alloc(size); + return kphp::memory::details::malloc_interface::alloc(size); } inline auto alloc_aligned(size_t size, std::align_val_t alignment) noexcept -> void* { - return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); + return kphp::memory::details::malloc_interface::alloc_aligned(size, alignment); } inline auto calloc(size_t num, size_t size) noexcept -> void* { - return kphp::memory::details::malloc_interface::calloc(num, size); + return kphp::memory::details::malloc_interface::calloc(num, size); } inline auto free(void* ptr) noexcept -> void { - kphp::memory::details::malloc_interface::free(ptr); + kphp::memory::details::malloc_interface::free(ptr); } inline auto realloc(void* ptr, size_t new_size) noexcept -> void* { - return kphp::memory::details::malloc_interface::realloc(ptr, new_size); + return kphp::memory::details::malloc_interface::realloc(ptr, new_size); } inline auto strdup(const char* str1) noexcept -> char* { - return kphp::memory::details::malloc_interface::strdup(str1); + return kphp::memory::details::malloc_interface::strdup(str1); } } // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp new file mode 100644 index 0000000000..4937eee417 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.cpp @@ -0,0 +1,15 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" + +#include "runtime-light/coroutine/coroutine-state.h" + +namespace kphp::coro::detail::memory { + +auto runtime_coroutine_allocator::get() noexcept -> runtime_coroutine_allocator& { + return kphp::coro::instance_state::get().coroutine_allocator; +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h new file mode 100644 index 0000000000..99f0cc0fd4 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h @@ -0,0 +1,44 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "runtime-common/core/allocator/pool-allocator.h" + +namespace kphp::coro::detail::memory { + +struct runtime_coroutine_allocator final { +private: + kphp::memory::pool_allocator m_allocator; + +public: + static auto get() noexcept -> runtime_coroutine_allocator&; + + runtime_coroutine_allocator(size_t script_mem_size, size_t min_extra_mem_size, size_t oom_handling_mem_size) noexcept + : m_allocator{script_mem_size, min_extra_mem_size, oom_handling_mem_size} {} + + auto free() noexcept -> void { + m_allocator.free(); + } + + auto alloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.alloc_script_memory(size); + } + + auto calloc_script_memory(size_t size) noexcept -> void* { + return m_allocator.calloc_script_memory(size); + } + + auto realloc_script_memory(void* mem, size_t new_size, size_t old_size) noexcept -> void* { + return m_allocator.realloc_script_memory(mem, new_size, old_size); + } + + auto free_script_memory(void* mem, size_t size) noexcept -> void { + m_allocator.free_script_memory(mem, size); + } +}; + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.cpp b/runtime-light/coroutine/detail/allocator/task-allocator.cpp new file mode 100644 index 0000000000..82d6a2e3b5 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/task-allocator.cpp @@ -0,0 +1,15 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" + +#include "runtime-light/coroutine/coroutine-state.h" + +namespace kphp::coro::detail::memory { + +auto task_allocator::get() noexcept -> task_allocator& { + return kphp::coro::instance_state::get().task_allocator; +} + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-allocator.h b/runtime-light/coroutine/detail/allocator/task-allocator.h new file mode 100644 index 0000000000..06285a714f --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/task-allocator.h @@ -0,0 +1,126 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include + +#include "common/containers/object-pool.h" +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/allocator/platform-malloc-interface.h" +#include "runtime-common/core/allocator/script-allocator.h" +#include "runtime-common/core/memory-resource/chunk-pool-resource.h" +#include "runtime-common/core/memory-resource/stack-resource.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::coro::detail::memory { + +namespace task { + +inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void*; +inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void; + +} // namespace task + +struct task_allocator final : private vk::not_copyable { +private: + vk::object_pool> m_stack_pool; + memory_resource::chunk_pool_resource m_chunk_pool; + memory_resource::stack_resource* m_curr_stack{nullptr}; + size_t m_stack_size{0}; + size_t m_min_extra_mem_size{0}; + + auto request_extra_memory(size_t requested_size) noexcept -> void { + size_t extra_mem_size{std::max(m_min_extra_mem_size, requested_size)}; + // Take into account internal layout of header for buffer + extra_mem_size += sizeof(memory_resource::chunk_pool_resource::buffer_header_size()); + // The smallest power of two that is not smaller than `extra_mem_size` + extra_mem_size = std::bit_ceil(extra_mem_size); + + auto* extra_mem{kphp::memory::platform::alloc(extra_mem_size)}; + + kphp::log::assertion(extra_mem != nullptr); + + m_chunk_pool.add_extra_memory(extra_mem, extra_mem_size); + } + +public: + static auto get() noexcept -> task_allocator&; + + task_allocator(size_t script_mem_size, size_t stack_size, size_t stack_pool_chunk_size, size_t min_extra_mem_size, size_t /*unused*/) noexcept + : m_stack_pool{stack_pool_chunk_size}, + m_stack_size{stack_size}, + m_min_extra_mem_size{min_extra_mem_size} { + // `acquire_stack` relies on the pool never returning nullptr + kphp::log::assertion(stack_pool_chunk_size > 0); + + void* buffer{kphp::memory::platform::alloc(script_mem_size)}; + + kphp::log::assertion(buffer != nullptr); + + m_chunk_pool.init(buffer, script_mem_size, m_stack_size); + } + + auto free() noexcept -> void { + auto* curr_buffer{m_chunk_pool.get_buffer_list_head()}; + while (curr_buffer != nullptr) { + auto* next_buffer = curr_buffer->next; + kphp::memory::platform::free(curr_buffer); + curr_buffer = next_buffer; + } + } + + auto current_stack() const noexcept -> memory_resource::stack_resource* { + return m_curr_stack; + } + + auto set_stack(memory_resource::stack_resource* stack) noexcept -> void { + m_curr_stack = stack; + } + + auto exchange_stack(memory_resource::stack_resource* stack) noexcept -> memory_resource::stack_resource* { + auto* prev{m_curr_stack}; + set_stack(stack); + + return prev; + } + +private: + auto stack_size() const noexcept -> size_t { + return m_stack_size; + } + + auto acquire_stack() noexcept -> memory_resource::stack_resource* { + auto* stack{m_stack_pool.acquire()}; + auto* chunk{m_chunk_pool.allocate()}; + if (chunk == nullptr) [[unlikely]] { + request_extra_memory(m_stack_size); + chunk = m_chunk_pool.allocate(); + + kphp::log::assertion(chunk != nullptr); + } + + stack->init(chunk, m_stack_size); + + return stack; + } + + auto release_stack(memory_resource::stack_resource* stack) noexcept -> void { + m_chunk_pool.deallocate(stack->memory_begin()); + m_stack_pool.release(stack); + } + + auto alloc_script_memory(size_t size) noexcept -> void* { + return m_curr_stack->allocate(size); + } + + auto free_script_memory(void* mem, size_t size) noexcept -> void { + m_curr_stack->deallocate(mem, size); + } + + friend inline auto kphp::coro::detail::memory::task::alloc_aligned(size_t size, std::align_val_t al) noexcept -> void*; + friend inline auto kphp::coro::detail::memory::task::free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void; +}; + +} // namespace kphp::coro::detail::memory diff --git a/runtime-light/coroutine/detail/allocator/task-malloc-interface.h b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h new file mode 100644 index 0000000000..d835c4c515 --- /dev/null +++ b/runtime-light/coroutine/detail/allocator/task-malloc-interface.h @@ -0,0 +1,98 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include +#include +#include +#include + +#include "runtime-common/core/memory-resource/stack-resource.h" +#include "runtime-light/coroutine/detail/allocator/runtime-coroutine-allocator.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +namespace kphp::coro::detail::memory::task { + +struct control_block { +public: + enum class backend_type : uint8_t { stack, stack_owner, coroutine_pool }; + + memory_resource::stack_resource* stack{nullptr}; + uint16_t base_offset{}; + backend_type backend{}; + + static constexpr auto max_alignment() noexcept -> uint64_t { + return std::numeric_limits::max(); + } +}; + +inline auto alloc_aligned(size_t size, std::align_val_t al) noexcept -> void* { + const size_t align{static_cast(al)}; + if (align >= kphp::coro::detail::memory::task::control_block::max_alignment()) [[unlikely]] { + php_warning("allocation alignment have to be not greater than %" PRIu64 ", got : %lu", kphp::coro::detail::memory::task::control_block::max_alignment(), + align); + return nullptr; + } + + constexpr size_t cb_size{sizeof(kphp::coro::detail::memory::task::control_block)}; + const size_t total_size{size + (align - 1) + cb_size}; + void* base{nullptr}; + kphp::coro::detail::memory::task::control_block::backend_type backend{}; + auto& task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + if (total_size <= task_allocator.stack_size()) [[likely]] { + if (task_allocator.current_stack() == nullptr) { + task_allocator.set_stack(task_allocator.acquire_stack()); + backend = kphp::coro::detail::memory::task::control_block::backend_type::stack_owner; + } else { + backend = kphp::coro::detail::memory::task::control_block::backend_type::stack; + } + + base = task_allocator.alloc_script_memory(total_size); + } + + if (base == nullptr) { + base = kphp::coro::detail::memory::runtime_coroutine_allocator::get().alloc_script_memory(total_size); + backend = kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool; + } + + const uint64_t base_u{reinterpret_cast(base)}; + // The smallest multiple of `align` greater than or equal to requested memory + const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)}; + const uint64_t base_offset_u{aligned_u - base_u}; + + std::construct_at(reinterpret_cast(aligned_u - cb_size), task_allocator.current_stack(), // NOLINT + static_cast(base_offset_u), backend); + + return reinterpret_cast(aligned_u); // NOLINT +} + +inline auto free_aligned(void* ptr, size_t size, std::align_val_t al) noexcept -> void { + const size_t align{static_cast(al)}; + const size_t cb_size{sizeof(kphp::coro::detail::memory::task::control_block)}; + const size_t total_size{size + (align - 1) + cb_size}; + auto* cb{reinterpret_cast(static_cast(ptr) - cb_size)}; + auto& task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + switch (cb->backend) { + case kphp::coro::detail::memory::task::control_block::backend_type::stack: + kphp::log::assertion(task_allocator.current_stack() == cb->stack); + + task_allocator.free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); + break; + case kphp::coro::detail::memory::task::control_block::backend_type::stack_owner: + kphp::log::assertion(task_allocator.current_stack() == cb->stack); + + task_allocator.free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); + task_allocator.set_stack(nullptr); + task_allocator.release_stack(cb->stack); + break; + case kphp::coro::detail::memory::task::control_block::backend_type::coroutine_pool: + kphp::coro::detail::memory::runtime_coroutine_allocator::get().free_script_memory(reinterpret_cast(ptr) - cb->base_offset, total_size); + break; + } +} + +} // namespace kphp::coro::detail::memory::task diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index b45f177559..c5f315afbf 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -9,10 +9,15 @@ #include #include #include +#include #include "common/containers/intrusive-list.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -41,6 +46,7 @@ class await_broker { vk::intrusive::list>> m_awaiters; await_set_ready_task_element* m_ready_tasks{}; size_t m_tasks_count{0}; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; public: await_broker() noexcept = default; @@ -82,7 +88,7 @@ class await_broker { * (in the future invariant [1] may not work). */ m_awaiters.pop_front(); - coroutine.resume(); + kphp::coro::resume(coroutine, m_task_allocator); } } @@ -116,7 +122,7 @@ class await_broker { */ m_tasks_storage.erase(task_iterator); --m_tasks_count; - typed_handle.destroy(); + kphp::coro::destroy(typed_handle, m_task_allocator); return result_t{std::move(result)}; } @@ -125,14 +131,14 @@ class await_broker { m_ready_tasks = nullptr; detach_all(); while (!m_tasks_storage.empty()) { - auto coroutine{m_tasks_storage.front()}; + auto coroutine{std::coroutine_handle::promise_type>::from_address(m_tasks_storage.front().address())}; /* * We can remove pop_front() here, because list node will be destroyed after destruction of coroutine frame and in its * destructor will call unlink() [1]. But we left pop_front() for better readability and safety * (in the future invariant [1] may not work). */ m_tasks_storage.pop_front(); - coroutine.destroy(); + kphp::coro::destroy(coroutine, m_task_allocator); } m_tasks_count = 0; @@ -150,7 +156,7 @@ class await_broker { * (in the future invariant [1] may not work). */ awaiters.pop_front(); - coroutine.resume(); + kphp::coro::resume(coroutine, m_task_allocator); } } @@ -158,6 +164,10 @@ class await_broker { return m_tasks_count; } + kphp::coro::detail::memory::task_allocator& task_allocator() noexcept { + return m_task_allocator; + } + ~await_broker() { abort_all(); } @@ -192,7 +202,10 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { } auto final_suspend() const noexcept { - struct completion_notifier { + struct completion_notifier : private kphp::coro::task_allocator_guard { + + explicit completion_notifier(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard(task_allocator) {} constexpr auto await_ready() const noexcept { return false; @@ -207,7 +220,8 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return completion_notifier{}; + + return completion_notifier{m_await_broker->get().task_allocator()}; } void unhandled_exception() const noexcept { @@ -231,7 +245,7 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_await_broker->get().task_allocator()); } }; @@ -314,14 +328,15 @@ class await_set_awaitable { private: await_broker& m_await_broker; - class awaiter { + class awaiter : private kphp::coro::task_allocator_guard { vk::intrusive::list_node> m_awaiting_coroutine_node; await_broker& m_await_broker; kphp::coro::async_stack_frame* caller_frame{}; public: explicit awaiter(await_broker& await_broker) noexcept - : m_await_broker(await_broker) {} + : kphp::coro::task_allocator_guard(await_broker.task_allocator()), + m_await_broker(await_broker) {} awaiter(awaiter&& other) noexcept = delete; awaiter(const awaiter& other) = delete; @@ -372,7 +387,7 @@ class await_set_awaitable { ~await_set_awaitable() = default; }; -template +template auto make_await_set_task(awaitable_type coroutine) noexcept -> await_set_task::awaiter_return_type> { if constexpr (std::is_void_v::awaiter_return_type>) { co_await std::move(coroutine); diff --git a/runtime-light/coroutine/detail/poll-info.h b/runtime-light/coroutine/detail/poll-info.h index 30c153a629..9b2c68edc4 100644 --- a/runtime-light/coroutine/detail/poll-info.h +++ b/runtime-light/coroutine/detail/poll-info.h @@ -11,7 +11,9 @@ #include "common/containers/intrusive-list.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/poll.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/k2-platform/k2-api.h" namespace kphp::coro::detail { @@ -39,9 +41,12 @@ struct poll_info { kphp::coro::poll_status m_poll_status{kphp::coro::poll_status::error}; kphp::coro::poll_op m_poll_op; - poll_info(k2::descriptor descriptor, kphp::coro::poll_op poll_op) noexcept + kphp::coro::detail::memory::task_allocator& m_task_allocator; + + poll_info(k2::descriptor descriptor, kphp::coro::poll_op poll_op, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept : m_descriptor(descriptor), - m_poll_op(poll_op) {} + m_poll_op(poll_op), + m_task_allocator(task_allocator) {} ~poll_info() = default; @@ -51,11 +56,12 @@ struct poll_info { poll_info& operator=(detail::poll_info&&) = delete; auto operator co_await() noexcept { - struct poll_awaiter { + struct poll_awaiter : private kphp::coro::task_allocator_guard { detail::poll_info& m_poll_info; explicit poll_awaiter(detail::poll_info& poll_info) noexcept - : m_poll_info(poll_info) {} + : kphp::coro::task_allocator_guard(poll_info.m_task_allocator), + m_poll_info(poll_info) {} constexpr auto await_ready() const noexcept -> bool { return false; diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index ec8ddea722..1d0bd148ea 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -107,7 +107,7 @@ inline auto promise_self_deleting::get_return_object_on_allocation_failure() noe } // namespace task_self_deleting -template +template auto make_task_self_deleting(awaitable_type awaitable) noexcept -> task_self_deleting::task_self_deleting { co_await std::move(awaitable); co_return; diff --git a/runtime-light/coroutine/detail/type-traits-utils.h b/runtime-light/coroutine/detail/type-traits-utils.h new file mode 100644 index 0000000000..5adaf3fc9b --- /dev/null +++ b/runtime-light/coroutine/detail/type-traits-utils.h @@ -0,0 +1,58 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include +#include + +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/shared-task.h" +#include "runtime-light/coroutine/task.h" + +namespace kphp::coro { + +namespace detail { + +template +requires std::invocable +class async_function_return_type { + using return_type = std::invoke_result_t; + + template + struct task_inner { + using type = U; + }; + + template + struct task_inner> { + using type = U; + }; + + template + struct task_inner> { + using type = U; + }; + + template + struct task_inner> { + using type = U; + }; + +public: + using type = task_inner::type; +}; + +template +struct is_task : std::false_type {}; + +template +struct is_task> : std::true_type {}; + +} // namespace detail + +template +inline constexpr bool is_task_v = kphp::coro::detail::is_task::value; + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index 186ba2793c..c214e0cfe1 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -16,7 +16,10 @@ #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -26,6 +29,7 @@ namespace kphp::coro::detail::when_all { class when_all_latch { size_t m_count{1}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; public: explicit when_all_latch(size_t count) noexcept @@ -33,7 +37,8 @@ class when_all_latch { when_all_latch(when_all_latch&& other) noexcept : m_count(std::exchange(other.m_count, 1)), - m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})) {} + m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})), + m_task_allocator(other.m_task_allocator) {} auto operator=(when_all_latch&& other) noexcept -> when_all_latch& { if (this != std::addressof(other)) { @@ -59,9 +64,13 @@ class when_all_latch { auto notify_awaitable_completed() noexcept -> void { if (--m_count == 1 && m_awaiting_coroutine != nullptr) { - m_awaiting_coroutine.resume(); + kphp::coro::resume(m_awaiting_coroutine, m_task_allocator); } } + + auto task_allocator() noexcept -> kphp::coro::detail::memory::task_allocator& { + return m_task_allocator; + } }; template @@ -89,13 +98,14 @@ class when_all_ready_awaitable> { when_all_latch m_latch; std::tuple m_tasks; - struct awaiter { + struct awaiter : private kphp::coro::task_allocator_guard { bool m_started{}; when_all_ready_awaitable& m_awaitable; kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; explicit awaiter(when_all_ready_awaitable& awaitable) noexcept - : m_awaitable(awaitable) {} + : kphp::coro::task_allocator_guard(awaitable.m_latch.task_allocator()), + m_awaitable(awaitable) {} auto await_ready() noexcept -> bool { kphp::log::assertion(!std::exchange(m_started, true)); // to make sure it's not co_awaited more than once @@ -120,7 +130,14 @@ class when_all_ready_awaitable> { kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); m_caller_async_stack_frame->async_stack_root->top_async_stack_frame = m_caller_async_stack_frame; } - return std::apply([](task_types&&... tasks) noexcept { return std::make_tuple(std::move(tasks).result()...); }, std::move(m_awaitable.m_tasks)); + + return std::apply( + [&latch = m_awaitable.m_latch](task_types&&... tasks) noexcept { + auto result{std::make_tuple(std::move(tasks).result()...)}; + (tasks.reset(latch.task_allocator()), ...); + return std::move(result); + }, + std::move(m_awaitable.m_tasks)); } }; @@ -170,7 +187,10 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { } auto final_suspend() const noexcept { - struct completion_notifier { + struct completion_notifier : private kphp::coro::task_allocator_guard { + explicit completion_notifier(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard(task_allocator) {} + constexpr auto await_ready() const noexcept { return false; } @@ -181,7 +201,8 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return completion_notifier{}; + + return completion_notifier{m_latch->task_allocator()}; } auto unhandled_exception() const noexcept -> void { @@ -198,7 +219,7 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), latch.task_allocator()); } }; @@ -270,7 +291,7 @@ class when_all_task { ~when_all_task() { if (m_coroutine != nullptr) { - m_coroutine.destroy(); + kphp::coro::destroy(m_coroutine, kphp::coro::detail::memory::task_allocator::get()); } } @@ -281,9 +302,15 @@ class when_all_task { auto result() && noexcept { return m_coroutine.promise().result(); } + + auto reset(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept -> void { + if (m_coroutine != nullptr) { + kphp::coro::destroy(std::exchange(m_coroutine, nullptr), task_allocator); + } + } }; -template +template auto make_when_all_task(awaitable_type awaitable) noexcept -> when_all_task::awaiter_return_type> { if constexpr (std::is_void_v::awaiter_return_type>) { co_await std::move(awaitable); diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index 8a90b1961f..00be6508a4 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -13,8 +13,12 @@ #include #include +#include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" #include "runtime-light/metaprogramming/type-functions.h" @@ -25,13 +29,15 @@ namespace kphp::coro::detail::when_any { class when_any_latch { bool m_toggled{}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; public: when_any_latch() noexcept = default; when_any_latch(when_any_latch&& other) noexcept : m_toggled(std::exchange(other.m_toggled, false)), - m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})) {} + m_awaiting_coroutine(std::exchange(other.m_awaiting_coroutine, {})), + m_task_allocator(other.m_task_allocator) {} auto operator=(when_any_latch&& other) noexcept -> when_any_latch& { if (this != std::addressof(other)) { @@ -58,9 +64,13 @@ class when_any_latch { auto notify_awaitable_completed() noexcept -> void { m_toggled = true; if (m_awaiting_coroutine != nullptr) { - m_awaiting_coroutine.resume(); + kphp::coro::resume(m_awaiting_coroutine, m_task_allocator); } } + + auto task_allocator() noexcept -> kphp::coro::detail::memory::task_allocator& { + return m_task_allocator; + } }; template @@ -89,13 +99,14 @@ class when_any_ready_awaitable> { std::tuple m_tasks; std::optional> m_result; - struct awaiter { + struct awaiter : private kphp::coro::task_allocator_guard { bool m_started{}; when_any_ready_awaitable& m_awaitable; kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; explicit awaiter(when_any_ready_awaitable& awaitable) noexcept - : m_awaitable(awaitable) {} + : kphp::coro::task_allocator_guard(awaitable.m_latch.task_allocator()), + m_awaitable(awaitable) {} auto await_ready() noexcept -> bool { kphp::log::assertion(!std::exchange(m_started, true)); // to make sure it's not co_awaited more than once @@ -129,9 +140,14 @@ class when_any_ready_awaitable> { result_variant_type{std::in_place_index()>, *std::move(task_result)}; } }}; - std::apply([&task_result_processor](auto&&... tasks) noexcept { (std::invoke(task_result_processor, std::forward(tasks)), ...); }, - std::move(m_awaitable.m_tasks)); + std::apply( + [&task_result_processor, &latch = m_awaitable.m_latch](auto&&... tasks) noexcept { + (std::invoke(task_result_processor, std::forward(tasks)), ...); + (tasks.reset(latch.task_allocator()), ...); + }, + std::move(m_awaitable.m_tasks)); kphp::log::assertion(m_awaitable.m_result.has_value()); + return std::move(*m_awaitable.m_result); } }; @@ -180,7 +196,10 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { } auto final_suspend() const noexcept { - struct completion_notifier { + struct completion_notifier : private kphp::coro::task_allocator_guard { + explicit completion_notifier(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard(task_allocator) {} + constexpr auto await_ready() const noexcept { return false; } @@ -191,7 +210,8 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return completion_notifier{}; + + return completion_notifier{m_latch->task_allocator()}; } auto unhandled_exception() const noexcept -> void { @@ -208,7 +228,7 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; async_stack_frame.return_address = return_address; async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + kphp::coro::resume(std::coroutine_handle::from_promise(*static_cast(this)), m_latch->task_allocator()); } }; @@ -291,7 +311,7 @@ class when_any_task { ~when_any_task() { if (m_coroutine != nullptr) { - m_coroutine.destroy(); + kphp::coro::destroy(m_coroutine, kphp::coro::detail::memory::task_allocator::get()); } } @@ -302,9 +322,15 @@ class when_any_task { auto result() && noexcept { return m_coroutine.promise().result(); } + + auto reset(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept -> void { + if (m_coroutine != nullptr) { + kphp::coro::destroy(std::exchange(m_coroutine, nullptr), task_allocator); + } + } }; -template +template auto make_when_any_task(awaitable_type awaitable) noexcept -> when_any_task::awaiter_return_type> { if constexpr (std::is_void_v::awaiter_return_type>) { co_await std::move(awaitable); diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index c8d7a98479..dbd26ee6a8 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -15,7 +15,10 @@ #include "common/wrappers/overloaded.h" #include "runtime-common/core/allocator/script-allocator-managed.h" #include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { @@ -26,13 +29,14 @@ class event { // 2) non empty list => linked list of coroutines waiting for the event to trigger // 3) empty list => the event is triggered and all coroutines are resumed std::variant>>> m_state; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; auto set() noexcept -> void; auto unset() noexcept -> void; auto is_set() const noexcept -> bool; }; - struct awaiter { + struct awaiter : private kphp::coro::task_allocator_guard { vk::intrusive::list_node> m_awaiting_coroutine_node; bool m_suspended{}; event_controller& m_controller; @@ -40,7 +44,8 @@ class event { kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; explicit awaiter(event_controller& event_controller) noexcept - : m_controller(event_controller), + : kphp::coro::task_allocator_guard(event_controller.m_task_allocator), + m_controller(event_controller), m_async_stack_root(kphp::coro::instance_state::get().coroutine_stack_root) {} awaiter(const awaiter&) = delete; @@ -131,7 +136,7 @@ inline auto event::event_controller::set() noexcept -> void { * (in the future invariant [1] may not work). */ awaiters.pop_front(); - coroutine.resume(); + kphp::coro::resume(coroutine, m_task_allocator); } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 2cc9f01cf8..6d3b00058c 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -26,11 +26,14 @@ #include "runtime-common/core/std/containers.h" #include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/control-functions.h" #include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" #include "runtime-light/coroutine/detail/poll-info.h" #include "runtime-light/coroutine/detail/task-self-deleting.h" #include "runtime-light/coroutine/detail/timer-handle.h" #include "runtime-light/coroutine/poll.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/when-any.h" @@ -96,7 +99,7 @@ class io_scheduler { * @param coroutine The coroutine to spawn. * @return True if the coroutine was successfully spawned, false otherwise. */ - template + template auto spawn(coroutine_type coroutine) noexcept -> bool; /** @@ -104,7 +107,7 @@ class io_scheduler { * @param corotuine The coroutine to start. * @return True if the coroutine was successfully started, false otherwise. */ - template + template auto start(coroutine_type coroutine) noexcept -> bool; /** @@ -126,7 +129,7 @@ class io_scheduler { * @param coroutine The coroutine to schedule. * @return A task that will yield the coroutine's return value when completed. */ - template + template [[nodiscard]] auto schedule(coroutine_type coroutine) noexcept -> kphp::coro::task::return_type>; /** * @brief Schedules a coroutine with a timeout. @@ -136,7 +139,7 @@ class io_scheduler { * - Positive: rounded up to the next whole millisecond. * @return A task that yields either the coroutine's result or kphp::coro::timeout_status::timeout. */ - template + template [[nodiscard]] auto schedule(coroutine_type coroutine, duration_type timeout) noexcept -> kphp::coro::task::return_type, timeout_status>>; @@ -436,14 +439,14 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { */ scheduled_coroutines.pop_front(); kphp::log::assertion(static_cast(coroutine)); - kphp::coro::resume(coroutine, m_coro_instance_state.coroutine_stack_root); + kphp::coro::resume(coroutine, m_coro_instance_state.coroutine_stack_root, m_coro_instance_state.task_allocator); } } return empty() ? k2::PollStatus::PollFinishedOk : k2::PollStatus::PollReschedule; } -template +template auto io_scheduler::spawn(coroutine_type coroutine) noexcept -> bool { auto owned_task{kphp::coro::detail::make_task_self_deleting(std::move(coroutine))}; auto& coroutine_node{owned_task.get_coroutine_node()}; @@ -454,14 +457,14 @@ auto io_scheduler::spawn(coroutine_type coroutine) noexcept -> bool { return true; } -template +template auto io_scheduler::start(coroutine_type coroutine) noexcept -> bool { auto owned_task{kphp::coro::detail::make_task_self_deleting(std::move(coroutine))}; auto handle{owned_task.get_handle()}; if (!handle || handle.done()) [[unlikely]] { return false; } - kphp::coro::resume(handle, m_coro_instance_state.coroutine_stack_root); + kphp::coro::resume(handle, m_coro_instance_state.coroutine_stack_root, m_coro_instance_state.task_allocator); return true; } @@ -505,21 +508,36 @@ inline auto io_scheduler::schedule() noexcept { m_schedule_pos); } - constexpr auto await_ready() const noexcept -> bool { - return false; - } - - auto await_suspend(std::coroutine_handle<> coroutine) noexcept -> void { - m_awaiting_coroutine_node.value() = coroutine; - m_scheduler.m_scheduled_coroutines.push_back(m_awaiting_coroutine_node); - m_schedule_pos = std::prev(m_scheduler.m_scheduled_coroutines.end()); - } - - auto await_resume() noexcept -> void { - m_schedule_pos = std::monostate{}; - m_scheduler.m_coro_instance_state.coroutine_stack_root.top_async_stack_frame = m_async_stack_frame; + auto operator co_await() noexcept { + struct awaiter : private kphp::coro::task_allocator_guard { + private: + schedule_operation& m_schedule_operation; + + public: + explicit awaiter(schedule_operation& schedule_operation, kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : kphp::coro::task_allocator_guard{task_allocator}, + m_schedule_operation{schedule_operation} {} + + constexpr auto await_ready() const noexcept -> bool { + return false; + } + + auto await_suspend(std::coroutine_handle<> coroutine) noexcept -> void { + m_schedule_operation.m_awaiting_coroutine_node.value() = coroutine; + m_schedule_operation.m_scheduler.m_scheduled_coroutines.push_back(m_schedule_operation.m_awaiting_coroutine_node); + m_schedule_operation.m_schedule_pos = std::prev(m_schedule_operation.m_scheduler.m_scheduled_coroutines.end()); + } + + auto await_resume() noexcept -> void { + m_schedule_operation.m_schedule_pos = std::monostate{}; + m_schedule_operation.m_scheduler.m_coro_instance_state.coroutine_stack_root.top_async_stack_frame = m_schedule_operation.m_async_stack_frame; + } + }; + + return awaiter{*this, m_scheduler.m_coro_instance_state.task_allocator}; } }; + return schedule_operation{*this}; } @@ -530,19 +548,19 @@ auto io_scheduler::schedule(duration_type timeout) noexcept -> kphp::coro::task< } // poll_op is not actually used here - kphp::coro::detail::poll_info poll_info{k2::INVALID_PLATFORM_DESCRIPTOR, kphp::coro::poll_op::read}; + kphp::coro::detail::poll_info poll_info{k2::INVALID_PLATFORM_DESCRIPTOR, kphp::coro::poll_op::read, m_coro_instance_state.task_allocator}; const auto cancellation_handler{make_cancellation_handler(poll_info)}; poll_info.m_schedule_position = add_timer_token(std::chrono::ceil(timeout), poll_info); co_await poll_info; } -template +template auto io_scheduler::schedule(coroutine_type coroutine) noexcept -> kphp::coro::task::return_type> { co_await schedule(); co_return co_await std::move(coroutine); } -template +template auto io_scheduler::schedule(coroutine_type coroutine, duration_type timeout) noexcept -> kphp::coro::task::return_type, timeout_status>> { using expected_return_type = typename kphp::coro::coroutine_traits::return_type; @@ -556,7 +574,8 @@ auto io_scheduler::schedule(coroutine_type coroutine, duration_type timeout) noe } } - auto result{co_await kphp::coro::when_any(std::move(coroutine), make_timeout_task(std::chrono::ceil(timeout)))}; + auto result{co_await kphp::coro::when_any( + std::move(coroutine), kphp::coro::detach_task(&io_scheduler::make_timeout_task, this, std::chrono::ceil(timeout)))}; if (std::holds_alternative(result)) [[unlikely]] { co_return std::unexpected{std::move(std::get<1>(result))}; } @@ -609,7 +628,7 @@ auto io_scheduler::poll(k2::descriptor descriptor, kphp::coro::poll_op poll_op, break; } - kphp::coro::detail::poll_info poll_info{descriptor, poll_op}; + kphp::coro::detail::poll_info poll_info{descriptor, poll_op, m_coro_instance_state.task_allocator}; const auto cancellation_handler{make_cancellation_handler(poll_info)}; if (timeout <= duration_type::zero()) { poll_info.m_schedule_position = m_parked_polls.emplace(poll_info.m_descriptor, poll_info); @@ -629,7 +648,7 @@ auto io_scheduler::accept(duration_type timeout) noexcept -> kphp::coro::task -struct promise_base : kphp::coro::async_stack_element { +struct promise_base : public kphp::coro::async_stack_element { private: struct not_started_tag {}; struct done_tag {}; @@ -60,7 +63,7 @@ struct promise_base : kphp::coro::async_stack_element { } auto& async_stack_root{*promise.get_async_stack_frame().async_stack_root}; - kphp::coro::resume(coroutine, async_stack_root); + kphp::coro::resume(coroutine, async_stack_root, promise.task_allocator()); } } @@ -103,7 +106,7 @@ struct promise_base : kphp::coro::async_stack_element { m_state = vk::intrusive::list>>{}; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; auto& async_stack_root{*get_async_stack_frame().async_stack_root}; - kphp::coro::resume(handle, async_stack_root); + kphp::coro::resume(handle, async_stack_root, m_task_allocator); } // coroutine already completed, don't suspend @@ -123,6 +126,10 @@ struct promise_base : kphp::coro::async_stack_element { return m_refcnt-- != 1; } + auto task_allocator() noexcept -> kphp::coro::detail::memory::task_allocator& { + return m_task_allocator; + } + template auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { return kphp::coro::detail::memory::alloc(n); @@ -140,10 +147,11 @@ struct promise_base : kphp::coro::async_stack_element { private: uint32_t m_refcnt{1}; std::variant>>> m_state; + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; }; template -class awaiter_base { +class awaiter_base : private kphp::coro::task_allocator_guard { void set_async_top_frame(async_stack_frame& caller_frame, void* return_address) noexcept { /** * shared_task is the top of the stack for calls from it. @@ -170,10 +178,12 @@ class awaiter_base { public: explicit awaiter_base(std::coroutine_handle coro) noexcept - : m_coro(coro) {} + : kphp::coro::task_allocator_guard(coro.promise().task_allocator()), + m_coro(coro) {} awaiter_base(awaiter_base&& other) noexcept - : m_awaiting_coroutine_node(std::move(other.m_awaiting_coroutine_node)), + : kphp::coro::task_allocator_guard(std::move(other)), + m_awaiting_coroutine_node(std::move(other.m_awaiting_coroutine_node)), m_coro(std::exchange(other.m_coro, {})) {} awaiter_base(const awaiter_base& other) = delete; @@ -309,11 +319,24 @@ struct shared_task final { } auto when_ready() const noexcept { - using awaiter_base = shared_task_impl::awaiter_base; - struct awaiter final : public awaiter_base { - using awaiter_base::awaiter_base; + struct awaitable { + private: + std::coroutine_handle m_coro; + + public: + explicit awaitable(std::coroutine_handle coro) noexcept + : m_coro{coro} {} + + auto operator co_await() noexcept { + using awaiter_base = shared_task_impl::awaiter_base; + struct awaiter final : public awaiter_base { + using awaiter_base::awaiter_base; + }; + return awaiter{m_coro}; + } }; - return awaiter{std::coroutine_handle::from_address(m_haddress)}; + + return awaitable{std::coroutine_handle::from_address(m_haddress)}; } auto get_handle() const noexcept -> std::coroutine_handle { @@ -353,7 +376,7 @@ struct shared_task final { } auto coro{std::coroutine_handle::from_address(m_haddress)}; if (!coro.promise().detach()) { - coro.destroy(); + kphp::coro::destroy(coro, coro.promise().task_allocator()); } } diff --git a/runtime-light/coroutine/task-allocator-guard.h b/runtime-light/coroutine/task-allocator-guard.h new file mode 100644 index 0000000000..94e3867c50 --- /dev/null +++ b/runtime-light/coroutine/task-allocator-guard.h @@ -0,0 +1,31 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/stack-resource.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" + +namespace kphp::coro { + +class task_allocator_guard : private vk::not_copyable { +private: + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + memory_resource::stack_resource* m_stack{nullptr}; + +public: + explicit task_allocator_guard() noexcept + : m_stack{m_task_allocator.current_stack()} {} + + explicit task_allocator_guard(kphp::coro::detail::memory::task_allocator& task_allocator) noexcept + : m_task_allocator{task_allocator}, + m_stack{m_task_allocator.current_stack()} {} + + ~task_allocator_guard() { + m_task_allocator.set_stack(m_stack); + } +}; + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/task-allocator-scope.h b/runtime-light/coroutine/task-allocator-scope.h new file mode 100644 index 0000000000..27cdc18f0c --- /dev/null +++ b/runtime-light/coroutine/task-allocator-scope.h @@ -0,0 +1,31 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "common/mixin/not_copyable.h" +#include "runtime-common/core/memory-resource/stack-resource.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" + +namespace kphp::coro { + +class task_allocator_scope : private vk::not_copyable { +private: + kphp::coro::detail::memory::task_allocator& m_task_allocator{kphp::coro::detail::memory::task_allocator::get()}; + memory_resource::stack_resource* m_stack{nullptr}; + +public: + explicit task_allocator_scope(memory_resource::stack_resource* stack) noexcept + : m_stack{m_task_allocator.exchange_stack(stack)} {} + + explicit task_allocator_scope(kphp::coro::detail::memory::task_allocator& task_allocator, memory_resource::stack_resource* stack) noexcept + : m_task_allocator{task_allocator}, + m_stack{m_task_allocator.exchange_stack(stack)} {} + + ~task_allocator_scope() { + m_task_allocator.set_stack(m_stack); + } +}; + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 8e874c480c..54bf27d035 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -12,11 +12,17 @@ #include "common/containers/final_action.h" #include "runtime-light/coroutine/async-stack.h" -#include "runtime-light/coroutine/detail/allocator/coroutine-malloc-interface.h" +#include "runtime-light/coroutine/control-functions.h" +#include "runtime-light/coroutine/detail/allocator/task-allocator.h" +#include "runtime-light/coroutine/detail/allocator/task-malloc-interface.h" +#include "runtime-light/coroutine/task-allocator-guard.h" #include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { +template +struct detached_task; + namespace task_impl { template @@ -64,18 +70,13 @@ struct promise_base : kphp::coro::async_stack_element { return std::coroutine_handle::from_promise(*const_cast(static_cast(this))).done(); } - template - auto operator new(size_t n, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::coro::detail::memory::alloc(n); - } - template auto operator new(size_t n, std::align_val_t al, [[maybe_unused]] Args&&... args) noexcept -> void* { - return kphp::coro::detail::memory::alloc_aligned(n, al); + return kphp::coro::detail::memory::task::alloc_aligned(n, al); } - auto operator delete(void* ptr, [[maybe_unused]] size_t n) noexcept -> void { - kphp::coro::detail::memory::free(ptr); + auto operator delete(void* ptr, size_t n, std::align_val_t al) noexcept -> void { + kphp::coro::detail::memory::task::free_aligned(ptr, n, al); } void* m_next{}; @@ -163,16 +164,9 @@ struct task { : m_coro(coro) {} task(const task& other) noexcept = delete; - - task(task&& other) noexcept - : m_coro(std::exchange(other.m_coro, {})) {} - + task(task&& other) noexcept = delete; task& operator=(const task& other) noexcept = delete; - - task& operator=(task&& other) noexcept { - std::swap(m_coro, other.m_coro); - return *this; - } + task& operator=(task&& other) noexcept = delete; ~task() { if (m_coro) { @@ -229,21 +223,10 @@ struct task { return m_coro; } - // conversion functions - // - // erase type - explicit operator task<>() && noexcept { - return task<>{std::exchange(m_coro, {})}; - } - // restore erased type - template - requires(std::same_as) - explicit operator task() && noexcept { - return task{std::exchange(m_coro, {})}; - } - private: std::coroutine_handle m_coro{}; + + friend struct kphp::coro::detached_task; }; } // namespace kphp::coro diff --git a/runtime-light/coroutine/type-traits.h b/runtime-light/coroutine/type-traits.h index 8d85fd4a54..12393178af 100644 --- a/runtime-light/coroutine/type-traits.h +++ b/runtime-light/coroutine/type-traits.h @@ -9,52 +9,29 @@ #include #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/detail/type-traits-utils.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" namespace kphp::coro { -namespace type_traits_impl { - template -requires std::invocable -class async_function_return_type { - using return_type = std::invoke_result_t; - - template - struct task_inner { - using type = U; - }; - - template - struct task_inner> { - using type = U; - }; - - template - struct task_inner> { - using type = U; - }; - -public: - using type = task_inner::type; -}; - -} // namespace type_traits_impl +inline constexpr bool is_task_function_v = requires { requires std::invocable && is_task_v>; }; template -requires std::invocable -inline constexpr bool is_async_function_v = requires { - { static_cast>(std::declval>()) }; -} || requires { +inline constexpr bool is_async_function_v = is_task_function_v || requires { + requires std::invocable; { static_cast>(std::declval>()) }; +} || requires { + { static_cast>(std::declval>()) }; }; // ================================================================================================ template requires std::invocable -using async_function_return_type_t = kphp::coro::type_traits_impl::async_function_return_type::type; +using async_function_return_type_t = kphp::coro::detail::async_function_return_type::type; // ================================================================================================ diff --git a/runtime-light/coroutine/when-all.h b/runtime-light/coroutine/when-all.h index 0803c8758d..cc98439a31 100644 --- a/runtime-light/coroutine/when-all.h +++ b/runtime-light/coroutine/when-all.h @@ -13,7 +13,7 @@ namespace kphp::coro { -template +template [[nodiscard]] auto when_all(awaitable_types... awaitables) noexcept { return detail::when_all::when_all_ready_awaitable< std::tuple::awaiter_return_type>...>>{ diff --git a/runtime-light/coroutine/when-any.h b/runtime-light/coroutine/when-any.h index 2136436115..a38c70ea02 100644 --- a/runtime-light/coroutine/when-any.h +++ b/runtime-light/coroutine/when-any.h @@ -13,7 +13,7 @@ namespace kphp::coro { -template +template [[nodiscard]] auto when_any(awaitable_types... awaitables) noexcept { return detail::when_any::when_any_ready_awaitable< std::tuple::awaiter_return_type>...>>{ diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index c16371b353..cefb53d717 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -19,6 +19,7 @@ include(${RUNTIME_LIGHT_DIR}/allocator/allocator.cmake) include(${RUNTIME_LIGHT_DIR}/components/kphp/kphp.cmake) include(${RUNTIME_LIGHT_DIR}/core/core.cmake) include(${RUNTIME_LIGHT_DIR}/server/server.cmake) +include(${RUNTIME_LIGHT_DIR}/coroutine/coroutine.cmake) include(${RUNTIME_LIGHT_DIR}/stdlib/stdlib.cmake) include(${RUNTIME_LIGHT_DIR}/tl/tl.cmake) include(${RUNTIME_LIGHT_DIR}/memory-resource-impl/memory-resource-impl.cmake) diff --git a/runtime-light/server/http/http-server-state.h b/runtime-light/server/http/http-server-state.h index 13d0d1fa94..a8dc989990 100644 --- a/runtime-light/server/http/http-server-state.h +++ b/runtime-light/server/http/http-server-state.h @@ -70,7 +70,7 @@ struct HttpServerInstanceState final : private vk::not_copyable { kphp::http::response_state response_state{kphp::http::response_state::not_started}; // The headers_registered_callback function should only be invoked once - std::optional> headers_registered_callback; + std::optional> headers_registered_callback; kphp::stl::unordered_set, kphp::memory::script_allocator> multipart_temporary_files; diff --git a/runtime-light/stdlib/component/component-api.h b/runtime-light/stdlib/component/component-api.h index 9719c1fa5c..4e67d87e12 100644 --- a/runtime-light/stdlib/component/component-api.h +++ b/runtime-light/stdlib/component/component-api.h @@ -119,7 +119,8 @@ inline auto f$component_client_send_request(string name, string request) noexcep auto stream{std::move(*expected_stream)}; auto request_span{std::span{request.c_str(), request.size()}}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::send_request(stream, std::as_bytes(request_span)))}; !expected) [[unlikely]] { + if (auto expected{co_await kphp::forks::id_managed(kphp::component::send_request, std::reference_wrapper{stream}, std::as_bytes(request_span))}; !expected) + [[unlikely]] { co_return class_instance{}; } co_return make_instance(std::move(stream)); @@ -132,7 +133,9 @@ inline auto f$component_client_fetch_response(class_instance q } string response{}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_response(query.get()->stream(), kphp::component::read_ext::append(response)))}; + auto callback{kphp::component::read_ext::append(response)}; + if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_response, std::reference_wrapper{query.get()->stream()}, + std::move(callback))}; !expected) [[unlikely]] { co_return string{}; } @@ -142,7 +145,7 @@ inline auto f$component_client_fetch_response(class_instance q // === component query server interface =========================================================== inline auto f$component_server_accept_query() noexcept -> kphp::coro::task> { - auto opt_stream{co_await kphp::forks::id_managed(kphp::component::stream::accept())}; + auto opt_stream{co_await kphp::forks::id_managed([]() noexcept { return kphp::component::stream::accept(); })}; if (!opt_stream) [[unlikely]] { co_return class_instance{}; } @@ -156,7 +159,9 @@ inline auto f$component_server_fetch_request(class_instance qu } string request{}; - if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_request(query.get()->stream(), kphp::component::read_ext::append(request)))}; + auto callback{kphp::component::read_ext::append(request)}; + if (auto expected{co_await kphp::forks::id_managed(kphp::component::fetch_request, std::reference_wrapper{query.get()->stream()}, + std::move(callback))}; !expected) [[unlikely]] { co_return string{}; } @@ -171,5 +176,5 @@ inline auto f$component_server_send_response(class_instance qu auto& stream{query.get()->stream()}; auto response_span{std::span{response.c_str(), response.size()}}; - co_await kphp::forks::id_managed(kphp::component::send_response(stream, std::as_bytes(response_span))); + co_await kphp::forks::id_managed(kphp::component::send_response, std::reference_wrapper{stream}, std::as_bytes(response_span)); } diff --git a/runtime-light/stdlib/component/inter-component-session/client.h b/runtime-light/stdlib/component/inter-component-session/client.h index 934b958ed9..3283b0104b 100644 --- a/runtime-light/stdlib/component/inter-component-session/client.h +++ b/runtime-light/stdlib/component/inter-component-session/client.h @@ -110,7 +110,7 @@ class client final { // The protocol design assumes that interrupting the transfer in the middle of a frame leads to critical error. // Therefore, we need to write the request in a separate coroutine. // This technique prevents integrity violations when this coroutine is cancelled. - kphp::coro::io_scheduler::get().start(process_write(t, qid, payload)); + kphp::coro::io_scheduler::get().start(kphp::coro::detach_task(&writer::process_write, this, t, qid, payload)); co_await req_finish_notifier[qid]; kphp::log::assertion(req_status.contains(qid)); @@ -141,7 +141,7 @@ class client final { reader(const shared_ctx_type& _ctx, shared_transport_type _t) noexcept : ctx(_ctx), interrupter(reader::wait_until_interrupt(_ctx)), - runner(reader::run(_ctx, std::move(_t), interrupter)){}; + runner(reader::run(_ctx, std::move(_t), interrupter)) {} static auto run(shared_ctx_type ctx, shared_transport_type t, kphp::coro::shared_task interrupter) noexcept -> kphp::coro::shared_task { // Allocate buffer for header @@ -150,7 +150,8 @@ class client final { while (!ctx.get()->interrupted.is_set()) { // Read response header or interrupt - auto read_header_res{co_await kphp::coro::when_any(t.get()->stream.read(resp_header_buf), interrupter)}; + auto read_header_res{ + co_await kphp::coro::when_any(kphp::coro::detach_task(&kphp::component::stream::read, t.get()->stream, resp_header_buf), interrupter)}; // An interrupt has occurred if (std::holds_alternative(read_header_res)) [[unlikely]] { kphp::log::debug("reader has been interrupted"); diff --git a/runtime-light/stdlib/crypto/crypto-functions.cpp b/runtime-light/stdlib/crypto/crypto-functions.cpp index 9c83b45f3a..79f468468f 100644 --- a/runtime-light/stdlib/crypto/crypto-functions.cpp +++ b/runtime-light/stdlib/crypto/crypto-functions.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -52,7 +53,9 @@ kphp::coro::task> f$openssl_random_pseudo_bytes(int64_t length) auto stream{*std::move(expected_stream)}; kphp::stl::vector response{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -75,7 +78,9 @@ kphp::coro::task>> f$openssl_x509_parse(string data, bool auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -122,7 +127,9 @@ kphp::coro::task f$openssl_sign(string data, string& signature, string pri auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -149,7 +156,9 @@ kphp::coro::task f$openssl_verify(string data, string signature, string auto stream{*std::move(expected_stream)}; std::array response{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), response))) [[unlikely]] { + if (!co_await kphp::forks::id_managed([](kphp::component::stream& stream, std::span request, + std::span response) noexcept { return kphp::component::query(stream, request, response); }, + std::reference_wrapper{stream}, tls.view(), std::span{response})) [[unlikely]] { co_return 0; } @@ -331,7 +340,9 @@ kphp::coro::task> f$openssl_encrypt(string data, string method, auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -402,7 +413,9 @@ kphp::coro::task> f$openssl_decrypt(string data, string method, auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -428,7 +441,9 @@ kphp::coro::task> f$openssl_pkey_get_public(string key) noexcep auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -457,7 +472,9 @@ kphp::coro::task> f$openssl_pkey_get_private(string key, string auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -486,7 +503,9 @@ kphp::coro::task f$openssl_public_encrypt(string data, string& encrypted_d auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -526,7 +545,9 @@ kphp::coro::task f$openssl_private_decrypt(string data, string& decrypted_ auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } @@ -579,7 +600,9 @@ kphp::coro::task send_and_get_string(tl::storer tls, bool raw_output) no auto stream{*std::move(expected_stream)}; kphp::stl::vector response_bytes{}; - if (!co_await kphp::forks::id_managed(kphp::component::query(stream, tls.view(), kphp::component::read_ext::append(response_bytes)))) [[unlikely]] { + auto callback{kphp::component::read_ext::append(response_bytes)}; + if (!co_await kphp::forks::id_managed(kphp::component::query, std::reference_wrapper{stream}, tls.view(), std::move(callback))) + [[unlikely]] { co_return false; } diff --git a/runtime-light/stdlib/curl/curl-easy-functions.h b/runtime-light/stdlib/curl/curl-easy-functions.h index 558c06f88d..b8c8b24c88 100644 --- a/runtime-light/stdlib/curl/curl-easy-functions.h +++ b/runtime-light/stdlib/curl/curl-easy-functions.h @@ -11,6 +11,8 @@ #include #include "runtime-common/core/runtime-core.h" +#include "runtime-light/coroutine/detached-task.h" +#include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/curl/curl-context.h" #include "runtime-light/stdlib/curl/curl-state.h" #include "runtime-light/stdlib/curl/defs.h" @@ -23,7 +25,7 @@ #include "runtime-light/stdlib/web-transfer-lib/web-simple-transfer.h" inline auto f$curl_init(string url = string{""}) noexcept -> kphp::coro::task { - auto open_res{co_await kphp::forks::id_managed(kphp::web::simple::open(kphp::web::transfer_backend::CURL))}; + auto open_res{co_await kphp::forks::id_managed(kphp::web::simple::open, kphp::web::transfer_backend::CURL)}; if (!open_res.has_value()) [[unlikely]] { kphp::web::curl::print_warning("could not initialize a new curl easy handle", std::move(open_res.error())); co_return 0; @@ -410,7 +412,7 @@ inline auto f$curl_exec(kphp::web::curl::easy_type easy_id) noexcept -> kphp::co if (!curl_state.easy_ctx.has(easy_id)) { co_return false; } - auto res{co_await kphp::forks::id_managed(kphp::web::simple::perform(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::perform, kphp::web::simple::transfer{easy_id})}; auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; easy_ctx.has_been_executed = true; if (!res.has_value()) [[unlikely]] { @@ -431,7 +433,7 @@ inline auto f$curl_close(kphp::web::curl::easy_type easy_id) noexcept -> kphp::c co_return; } auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::simple::close(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::close, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not close curl easy handle", std::move(res.error())); @@ -444,7 +446,7 @@ inline auto f$curl_reset(kphp::web::curl::easy_type easy_id) noexcept -> kphp::c co_return; } auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::simple::reset(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::reset, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not reset curl easy handle", std::move(res.error())); @@ -473,8 +475,10 @@ inline auto f$curl_exec_concurrently(kphp::web::curl::easy_type easy_id, double timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; - auto sched_res{ - co_await kphp::coro::io_scheduler::get().schedule(kphp::forks::id_managed(kphp::web::simple::perform(kphp::web::simple::transfer{easy_id})), timeout)}; + auto sched_res{co_await kphp::coro::io_scheduler::get().schedule( + kphp::coro::detach_task([](kphp::web::simple::transfer easy_id) noexcept { return kphp::forks::id_managed(kphp::web::simple::perform, easy_id); }, + kphp::web::simple::transfer{easy_id}), + timeout)}; if (!sched_res.has_value()) [[unlikely]] { kphp::web::curl::print_debug( "could not execute curl easy handle concurrently", @@ -522,8 +526,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; switch (static_cast(option)) { case kphp::web::curl::CURLINFO::NONE: { - auto res{ - co_await kphp::forks::id_managed(kphp::web::property::get(kphp::web::simple::transfer{easy_id}, std::nullopt, kphp::web::property::get_policy::load))}; + auto res{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, std::nullopt, + kphp::web::property::get_policy::load)}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not get all info options of easy handle", std::move(res.error())); @@ -544,8 +548,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = const auto& image_state{CurlImageState::get()}; if (!easy_ctx.has_been_executed) { const auto url_opt_id{static_cast(kphp::web::curl::CURLOPT::URL)}; - const auto url{co_await kphp::forks::id_managed( - kphp::web::property::get(kphp::web::simple::transfer{easy_id}, url_opt_id, kphp::web::property::get_policy::cached))}; + const auto url{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, url_opt_id, + kphp::web::property::get_policy::cached)}; if (url.has_value()) { const auto& v{(*url).find(url_opt_id)}; kphp::log::assertion(v != (*url).end()); @@ -589,8 +593,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = case kphp::web::curl::CURLINFO::EFFECTIVE_URL: if (!easy_ctx.has_been_executed) { const auto url_opt_id{static_cast(kphp::web::curl::CURLOPT::URL)}; - const auto url{co_await kphp::forks::id_managed( - kphp::web::property::get(kphp::web::simple::transfer{easy_id}, url_opt_id, kphp::web::property::get_policy::cached))}; + const auto url{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, url_opt_id, + kphp::web::property::get_policy::cached)}; if (url.has_value()) { co_return (*url).find(url_opt_id)->second.to_mixed(); } @@ -622,7 +626,8 @@ inline auto f$curl_getinfo(kphp::web::curl::easy_type easy_id, int64_t option = case kphp::web::curl::CURLINFO::CONDITION_UNMET: case kphp::web::curl::CURLINFO::NUM_CONNECTS: case kphp::web::curl::CURLINFO::HEADER_OUT: { - auto res{co_await kphp::forks::id_managed(kphp::web::property::get(kphp::web::simple::transfer{easy_id}, option, kphp::web::property::get_policy::load))}; + auto res{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::simple::transfer{easy_id}, option, + kphp::web::property::get_policy::load)}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code, res.error().description); kphp::web::curl::print_warning("could not get a specific info of easy handle", std::move(res.error())); diff --git a/runtime-light/stdlib/curl/curl-multi-functions.h b/runtime-light/stdlib/curl/curl-multi-functions.h index c85a7f270f..d764d97308 100644 --- a/runtime-light/stdlib/curl/curl-multi-functions.h +++ b/runtime-light/stdlib/curl/curl-multi-functions.h @@ -23,7 +23,7 @@ #include "runtime-light/stdlib/web-transfer-lib/web-simple-transfer.h" inline auto f$curl_multi_init() noexcept -> kphp::coro::task { - auto open_res{co_await kphp::forks::id_managed(kphp::web::composite::open(kphp::web::transfer_backend::CURL))}; + auto open_res{co_await kphp::forks::id_managed(kphp::web::composite::open, kphp::web::transfer_backend::CURL)}; if (!open_res.has_value()) [[unlikely]] { kphp::web::curl::print_warning("could not initialize a new curl multi handle", std::move(open_res.error())); co_return 0; @@ -45,7 +45,7 @@ inline auto f$curl_multi_add_handle(kphp::web::curl::multi_type multi_id, kphp:: auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; easy_ctx.errors_reset(); - auto res{co_await kphp::forks::id_managed(kphp::web::composite::add(kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::add, kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not add a curl easy handler into multi handle", std::move(res.error())); @@ -65,7 +65,7 @@ inline auto f$curl_multi_remove_handle(kphp::web::curl::multi_type multi_id, if (!curl_state.easy_ctx.has(easy_id)) [[unlikely]] { co_return false; } - auto res{co_await kphp::forks::id_managed(kphp::web::composite::remove(kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::remove, kphp::web::composite::transfer{multi_id}, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not remove a curl easy handler from multi handle", std::move(res.error())); @@ -127,7 +127,7 @@ inline auto f$curl_multi_exec(kphp::web::curl::multi_type multi_id, int64_t& sti if (!curl_state.multi_ctx.has(multi_id)) [[unlikely]] { co_return false; } - auto res{co_await kphp::forks::id_managed(kphp::web::composite::perform(kphp::web::composite::transfer{multi_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::perform, kphp::web::composite::transfer{multi_id})}; auto& multi_ctx{curl_state.multi_ctx.get_or_init(multi_id)}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); @@ -146,7 +146,7 @@ inline auto f$curl_multi_getcontent(kphp::web::curl::easy_type easy_id) noexcept } auto& easy_ctx{curl_state.easy_ctx.get_or_init(easy_id)}; if (easy_ctx.return_transfer) { - auto res{co_await kphp::forks::id_managed(kphp::web::simple::get_response(kphp::web::simple::transfer{easy_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::simple::get_response, kphp::web::simple::transfer{easy_id})}; if (!res.has_value()) [[unlikely]] { easy_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not get response of curl easy handle", std::move(res.error())); @@ -163,7 +163,7 @@ inline auto f$curl_multi_close(kphp::web::curl::multi_type multi_id) noexcept -> co_return; } auto& multi_ctx{curl_state.multi_ctx.get_or_init(multi_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::composite::close(kphp::web::composite::transfer{multi_id}))}; + auto res{co_await kphp::forks::id_managed(kphp::web::composite::close, kphp::web::composite::transfer{multi_id})}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not close curl multi handle", std::move(res.error())); @@ -223,8 +223,11 @@ inline auto f$curl_multi_select(kphp::web::curl::multi_type multi_id, double tim co_return false; } auto& multi_ctx{curl_state.multi_ctx.get_or_init(multi_id)}; - auto res{co_await kphp::forks::id_managed(kphp::web::composite::wait_updates( - kphp::web::composite::transfer{multi_id}, std::chrono::duration_cast(std::chrono::duration{timeout})))}; + auto res{co_await kphp::forks::id_managed( + [](kphp::web::composite::transfer multi_id, std::chrono::seconds timeout) noexcept { + return kphp::web::composite::wait_updates(kphp::web::composite::transfer{multi_id}, timeout); + }, + kphp::web::composite::transfer{multi_id}, std::chrono::duration_cast(std::chrono::duration{timeout}))}; if (!res.has_value()) [[unlikely]] { multi_ctx.set_errno(res.error().code); kphp::web::curl::print_warning("could not select curl multi handle", std::move(res.error())); @@ -243,8 +246,8 @@ inline auto f$curl_multi_info_read(kphp::web::curl::multi_type multi_id, constexpr auto CURL_MULTI_INFO_READ_OPTION = 0; - auto props{co_await kphp::forks::id_managed( - kphp::web::property::get(kphp::web::composite::transfer{multi_id}, CURL_MULTI_INFO_READ_OPTION, kphp::web::property::get_policy::load))}; + auto props{co_await kphp::forks::id_managed(kphp::web::property::get, kphp::web::composite::transfer{multi_id}, + CURL_MULTI_INFO_READ_OPTION, kphp::web::property::get_policy::load)}; if (!props.has_value()) [[unlikely]] { multi_ctx.set_errno(props.error().code, props.error().description); kphp::web::curl::print_warning("could not get info message of multi handle", std::move(props.error())); diff --git a/runtime-light/stdlib/file/file-system-functions.h b/runtime-light/stdlib/file/file-system-functions.h index 8741257e5f..d99fbfd5f3 100644 --- a/runtime-light/stdlib/file/file-system-functions.h +++ b/runtime-light/stdlib/file/file-system-functions.h @@ -110,7 +110,7 @@ inline kphp::coro::task f$fclose(resource stream) noexcept { if (auto sync_resource{from_mixed>(stream, {})}; !sync_resource.is_null()) { co_return sync_resource.get()->close(); } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - co_return co_await kphp::forks::id_managed(async_resource.get()->close()); + co_return co_await kphp::forks::id_managed(&kphp::fs::async_resource::close, async_resource.get()); } kphp::log::warning("unexpected resource in fclose -> {}", stream.to_string().c_str()); @@ -133,7 +133,7 @@ inline kphp::coro::task> f$fwrite(resource stream, string data auto expected{sync_resource.get()->write(std::as_bytes(data_span))}; co_return expected ? Optional{static_cast(*std::move(expected))} : Optional{false}; } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - auto expected{co_await kphp::forks::id_managed(async_resource.get()->write(std::as_bytes(data_span)))}; + auto expected{co_await kphp::forks::id_managed(&kphp::fs::async_resource::write, async_resource.get(), std::as_bytes(data_span))}; co_return expected ? Optional{static_cast(*std::move(expected))} : Optional{false}; } @@ -152,7 +152,7 @@ inline kphp::coro::task> f$fread(resource stream, int64_t lengt if (auto sync_resource{from_mixed>(stream, {})}; !sync_resource.is_null()) { expected = sync_resource.get()->read(std::as_writable_bytes(buf)); } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - expected = co_await kphp::forks::id_managed(async_resource.get()->read(std::as_writable_bytes(buf))); + expected = co_await kphp::forks::id_managed(&kphp::fs::async_resource::read, async_resource.get(), std::as_writable_bytes(buf)); } else { kphp::log::warning("unexpected resource in fread -> {}", stream.to_string().c_str()); co_return false; @@ -168,7 +168,7 @@ inline kphp::coro::task f$fflush(resource stream) noexcept { if (auto sync_resource{from_mixed>(stream, {})}; !sync_resource.is_null()) { co_return sync_resource.get()->flush(); } else if (auto async_resource{from_mixed>(stream, {})}; !async_resource.is_null()) { - co_return co_await kphp::forks::id_managed(async_resource.get()->flush()); + co_return co_await kphp::forks::id_managed(&kphp::fs::async_resource::flush, async_resource.get()); } kphp::log::warning("unexpected resource in fflush -> {}", stream.to_string().c_str()); diff --git a/runtime-light/stdlib/file/resource.h b/runtime-light/stdlib/file/resource.h index 9acbc0e740..ee83c32957 100644 --- a/runtime-light/stdlib/file/resource.h +++ b/runtime-light/stdlib/file/resource.h @@ -23,6 +23,7 @@ #include "runtime-common/core/class-instance/refcountable-php-classes.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/k2-platform/k2-api.h" @@ -637,7 +638,7 @@ class socket : public async_resource { socket& operator=(socket&& other) noexcept { if (this != std::addressof(other)) { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(finalizer(std::move(m_stream), std::move(m_buf))); + kphp::coro::io_scheduler::get().spawn(kphp::coro::detach_task(&socket::finalizer, std::move(m_stream), std::move(m_buf))); } m_open = std::exchange(other.m_open, false); @@ -649,7 +650,7 @@ class socket : public async_resource { ~socket() override { if (!finalized()) { - kphp::coro::io_scheduler::get().spawn(finalizer(std::move(m_stream), std::move(m_buf))); + kphp::coro::io_scheduler::get().spawn(kphp::coro::detach_task(&socket::finalizer, std::move(m_stream), std::move(m_buf))); } } diff --git a/runtime-light/stdlib/fork/fork-functions.h b/runtime-light/stdlib/fork/fork-functions.h index fd2c4de625..edb6d83ec2 100644 --- a/runtime-light/stdlib/fork/fork-functions.h +++ b/runtime-light/stdlib/fork/fork-functions.h @@ -16,6 +16,7 @@ #include "runtime-common/core/core-types/decl/optional.h" #include "runtime-common/core/runtime-core.h" #include "runtime-light/coroutine/concepts.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" @@ -44,7 +45,7 @@ class scoped_id_managed { scoped_id_managed& operator=(scoped_id_managed&&) = delete; }; -template +template auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task::awaiter_return_type> { auto& fork_instance_st{ForkInstanceState::get()}; const auto saved_fork_id{fork_instance_st.current_id}; @@ -59,8 +60,24 @@ auto id_managed(awaitable_type awaitable) noexcept -> kphp::coro::task +requires(kphp::coro::is_task_function_v) +auto id_managed(F f, Args... args) noexcept -> kphp::coro::task>::awaiter_return_type> { + auto& fork_instance_st{ForkInstanceState::get()}; + const auto saved_fork_id{fork_instance_st.current_id}; + if constexpr (std::is_void_v>::awaiter_return_type>) { + co_await std::invoke(std::move(f), std::move(args)...); + fork_instance_st.current_id = saved_fork_id; + co_return; + } else { + auto value{co_await std::invoke(std::move(f), std::move(args)...)}; + fork_instance_st.current_id = saved_fork_id; + co_return std::move(value); + } +} + template -auto start(kphp::coro::task task) noexcept -> int64_t { +auto start(kphp::coro::detached_task task) noexcept -> int64_t { auto& fork_instance_st{ForkInstanceState::get()}; auto [fork_id, fork_task]{fork_instance_st.create_fork(std::move(task))}; auto saved_fork_id{fork_instance_st.current_id}; @@ -127,7 +144,8 @@ auto wait(int64_t fork_id, duration_type timeout) noexcept -> kphp::coro::task requires(is_optional::value || std::same_as || is_class_instance::value) kphp::coro::task f$wait(int64_t fork_id, double timeout = -1.0) noexcept { - auto opt_result{co_await kphp::forks::id_managed(kphp::forks::wait(fork_id, std::chrono::duration{timeout}))}; + auto opt_result{ + co_await kphp::forks::id_managed(kphp::forks::wait>, fork_id, std::chrono::duration{timeout})}; co_return opt_result ? return_type{*std::move(opt_result)} : return_type{}; } @@ -196,7 +214,8 @@ inline kphp::coro::task<> f$sched_yield() noexcept { } inline kphp::coro::task<> f$sched_yield_sleep(double duration) noexcept { - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(std::chrono::duration{duration})); + co_await kphp::forks::id_managed([](std::chrono::duration duration) noexcept { return kphp::coro::io_scheduler::get().schedule(duration); }, + std::chrono::duration{duration}); } // ================================================================================================ diff --git a/runtime-light/stdlib/fork/fork-state.h b/runtime-light/stdlib/fork/fork-state.h index d98eddccb5..d29e38dbba 100644 --- a/runtime-light/stdlib/fork/fork-state.h +++ b/runtime-light/stdlib/fork/fork-state.h @@ -13,8 +13,8 @@ #include "common/mixin/not_copyable.h" #include "runtime-common/core/allocator/script-allocator.h" #include "runtime-common/core/std/containers.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/shared-task.h" -#include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/exception-types.h" #include "runtime-light/stdlib/diagnostics/logs.h" #include "runtime-light/stdlib/fork/fork-storage.h" @@ -50,9 +50,9 @@ struct ForkInstanceState final : private vk::not_copyable { static ForkInstanceState& get() noexcept; template - std::pair> create_fork(kphp::coro::task task) noexcept { + std::pair> create_fork(kphp::coro::detached_task task) noexcept { static constexpr auto fork_coroutine{ - [](kphp::coro::task task, int64_t fork_id) noexcept -> kphp::coro::shared_task { + [](kphp::coro::detached_task task, int64_t fork_id) noexcept -> kphp::coro::shared_task { ForkInstanceState::get().current_id = fork_id; kphp::forks::details::storage s{}; diff --git a/runtime-light/stdlib/fork/wait-queue-functions.h b/runtime-light/stdlib/fork/wait-queue-functions.h index 1cfa8948c7..74e897996f 100644 --- a/runtime-light/stdlib/fork/wait-queue-functions.h +++ b/runtime-light/stdlib/fork/wait-queue-functions.h @@ -11,6 +11,7 @@ #include "runtime-common/core/core-types/decl/optional.h" #include "runtime-light/coroutine/await-set.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/task.h" #include "runtime-light/stdlib/diagnostics/logs.h" #include "runtime-light/stdlib/fork/fork-functions.h" @@ -56,7 +57,7 @@ inline kphp::coro::task> wait_queue_next(int64_t queue_id timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(wait_queue_next_task(await_set.next()), timeout)}; + auto wait_result{co_await kphp::coro::io_scheduler::get().schedule(kphp::coro::detach_task(wait_queue_next_task, await_set.next()), timeout)}; if (!wait_result) { co_return std::nullopt; } @@ -94,7 +95,7 @@ inline void wait_queue_push(int64_t queue_id, int64_t fork_id) noexcept { }}; auto& await_set{(*opt_await_set).get()}; - await_set.push(wait_queue_wrapper_task(std::move(fork_task), fork_id)); + await_set.push(kphp::coro::detach_task(wait_queue_wrapper_task, std::move(fork_task), fork_id)); } } // namespace kphp::forks @@ -118,8 +119,8 @@ inline bool f$wait_queue_empty(int64_t queue_id) noexcept { } inline kphp::coro::task> f$wait_queue_next(int64_t queue_id, double timeout = -1.0) noexcept { - auto opt_result{co_await kphp::forks::id_managed( - kphp::forks::wait_queue_next(queue_id, std::chrono::duration_cast(std::chrono::duration{timeout})))}; + auto opt_result{co_await kphp::forks::id_managed(kphp::forks::wait_queue_next, queue_id, + std::chrono::duration_cast(std::chrono::duration{timeout}))}; co_return opt_result ? *opt_result : Optional{false}; } diff --git a/runtime-light/stdlib/rpc/rpc-api.cpp b/runtime-light/stdlib/rpc/rpc-api.cpp index 5c573dc222..fce69b6643 100644 --- a/runtime-light/stdlib/rpc/rpc-api.cpp +++ b/runtime-light/stdlib/rpc/rpc-api.cpp @@ -20,6 +20,7 @@ #include "common/rpc-error-codes.h" #include "runtime-common/core/runtime-core.h" #include "runtime-light/allocator/allocator.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/io-scheduler.h" #include "runtime-light/coroutine/shared-task.h" #include "runtime-light/coroutine/task.h" @@ -358,7 +359,7 @@ kphp::rpc::query_info send_request(std::string_view actor, std::optional return {reinterpret_cast(response_exp->buffer()), size}; }}; - auto fetch_task{kphp::rpc::query::response(std::move(q), response_buffer_provider)}; + auto fetch_task{kphp::coro::detach_task(kphp::rpc::query::response, std::move(q), response_buffer_provider)}; auto fetch_result{co_await kphp::coro::io_scheduler::get().schedule(std::move(fetch_task))}; if (!fetch_result) [[unlikely]] { response_exp = std::unexpected{fetch_result.error()}; diff --git a/runtime-light/stdlib/rpc/rpc-api.h b/runtime-light/stdlib/rpc/rpc-api.h index 4e162f775b..db17c02bb1 100644 --- a/runtime-light/stdlib/rpc/rpc-api.h +++ b/runtime-light/stdlib/rpc/rpc-api.h @@ -277,7 +277,7 @@ inline kphp::coro::task f$store_error(int64_t error_code, string error_msg tl::storer tls{rpc_response.footprint()}; rpc_response.store(tls); - if (auto expected{co_await kphp::forks::id_managed(kphp::rpc::send_response(tls.view()))}; !expected) [[unlikely]] { + if (auto expected{co_await kphp::forks::id_managed(kphp::rpc::send_response, tls.view())}; !expected) [[unlikely]] { kphp::log::warning("can't store RPC error: {}", expected.error()); } kphp::log::error("store_error called. error_code: {}, error_msg: {}", error_code, error_msg.c_str()); @@ -300,7 +300,7 @@ inline kphp::coro::task<> f$rpc_server_store_response(class_instance f$rpc_send_requests(const string& actor, const array>> f$rpc_fetch_responses(array query_ids) noexcept { array> res{query_ids.size()}; for (const auto& it : std::as_const(query_ids)) { - res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::rpc_tl_query_result_one_impl(it.get_value()))); + res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::rpc_tl_query_result_one_impl, it.get_value())); } co_return std::move(res); } @@ -392,7 +392,7 @@ requires std::default_initializable kphp::coro::task>> f$rpc_fetch_typed_responses(array query_ids) noexcept { array> res{query_ids.size()}; for (const auto& it : std::as_const(query_ids)) { - res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::typed_rpc_tl_query_result_one_impl(it.get_value(), error_factory_type{}))); + res.set_value(it.get_key(), co_await kphp::forks::id_managed(kphp::rpc::detail::typed_rpc_tl_query_result_one_impl, it.get_value(), error_factory_type{})); } co_return std::move(res); } diff --git a/runtime-light/stdlib/rpc/rpc-queue-functions.h b/runtime-light/stdlib/rpc/rpc-queue-functions.h index d120f452fd..dcf2d961aa 100644 --- a/runtime-light/stdlib/rpc/rpc-queue-functions.h +++ b/runtime-light/stdlib/rpc/rpc-queue-functions.h @@ -43,7 +43,7 @@ inline void rpc_queue_push(int64_t queue_id, int64_t request_id) noexcept { } auto& await_set{(*opt_await_set).get()}; - await_set.push(rpc_queue_wrapper_task(static_cast>(it_awaiter_task->second), request_id)); + await_set.push(kphp::coro::detach_task(rpc_queue_wrapper_task, static_cast>(it_awaiter_task->second), request_id)); } inline int64_t rpc_queue_create(std::span request_ids) noexcept { @@ -83,7 +83,7 @@ inline kphp::coro::task> rpc_queue_next(int64_t queue_id, timeout = (std::clamp(timeout, duration_type::zero(), MAX_TIMEOUT) != timeout) ? DEFAULT_TIMEOUT : timeout; - const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(rpc_queue_next_task(await_set.next()), timeout)}; + const auto expected_next{co_await kphp::coro::io_scheduler::get().schedule(kphp::coro::detach_task(rpc_queue_next_task, await_set.next()), timeout)}; if (!expected_next) { co_return std::nullopt; } @@ -136,7 +136,7 @@ inline bool f$rpc_queue_empty(int64_t queue_id) noexcept { } inline kphp::coro::task> f$rpc_queue_next(int64_t queue_id, double timeout = -1) noexcept { - auto opt_result{co_await kphp::forks::id_managed( - kphp::rpc::rpc_queue_next(queue_id, std::chrono::duration_cast(std::chrono::duration{timeout})))}; + auto opt_result{co_await kphp::forks::id_managed(kphp::rpc::rpc_queue_next, queue_id, + std::chrono::duration_cast(std::chrono::duration{timeout}))}; co_return opt_result ? *opt_result : Optional{false}; } diff --git a/runtime-light/stdlib/server/common-functions.h b/runtime-light/stdlib/server/common-functions.h index f622501743..d9d8573da8 100644 --- a/runtime-light/stdlib/server/common-functions.h +++ b/runtime-light/stdlib/server/common-functions.h @@ -31,7 +31,7 @@ inline auto f$ignore_user_abort(Optional enable) noexcept -> kphp::coro::t http_server_instance_st.connection->decrease_ignore_abort_level(); if (http_server_instance_st.connection->get_ignore_abort_level() == 0 && http_server_instance_st.connection->is_aborted()) { - co_await kphp::forks::id_managed(kphp::system::exit(1)); + co_await kphp::forks::id_managed(kphp::system::exit, 1); } co_return prev; } diff --git a/runtime-light/stdlib/server/handler-functions.h b/runtime-light/stdlib/server/handler-functions.h index 41c9981b0c..c8708a3447 100644 --- a/runtime-light/stdlib/server/handler-functions.h +++ b/runtime-light/stdlib/server/handler-functions.h @@ -19,7 +19,7 @@ void f$register_shutdown_function(F&& f, Args&&... args) noexcept { // it's a lambda coroutine, so: // 1. don't capture anything; // 2. parameters are passed by value. - auto shutdown_function_task{std::invoke( + auto shutdown_function_task{kphp::coro::detach_task( [](F f, Args... args) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f), std::move(args)...); diff --git a/runtime-light/stdlib/server/http-functions.h b/runtime-light/stdlib/server/http-functions.h index cb4b9bb8ac..c74f27351d 100644 --- a/runtime-light/stdlib/server/http-functions.h +++ b/runtime-light/stdlib/server/http-functions.h @@ -75,7 +75,7 @@ bool f$header_register_callback(F&& f) noexcept { return false; } - auto headers_callback_task{std::invoke( + auto headers_callback_task{kphp::coro::detach_task( [](F f) noexcept -> kphp::coro::task<> { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(f)); diff --git a/runtime-light/stdlib/system/system-functions.h b/runtime-light/stdlib/system/system-functions.h index 09cad7370a..65d0661a47 100644 --- a/runtime-light/stdlib/system/system-functions.h +++ b/runtime-light/stdlib/system/system-functions.h @@ -95,7 +95,7 @@ inline kphp::coro::task<> f$exit(mixed v = 0) noexcept { // TODO: make it synchr } else { exit_code = 1; } - co_await kphp::forks::id_managed(kphp::system::exit(static_cast(exit_code))); + co_await kphp::forks::id_managed(kphp::system::exit, static_cast(exit_code)); } inline kphp::coro::task<> f$die(mixed v = 0) noexcept { @@ -221,7 +221,8 @@ inline kphp::coro::task<> f$usleep(int64_t microseconds) noexcept { kphp::log::warning("value of microseconds ({}) must be positive", microseconds); co_return; } - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(std::chrono::microseconds{microseconds})); + co_await kphp::forks::id_managed([](std::chrono::microseconds microseconds) noexcept { return kphp::coro::io_scheduler::get().schedule(microseconds); }, + std::chrono::microseconds{microseconds}); } inline kphp::coro::task<> f$sleep(int64_t seconds) noexcept { @@ -230,7 +231,8 @@ inline kphp::coro::task<> f$sleep(int64_t seconds) noexcept { co_return; } - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(std::chrono::seconds{seconds})); + co_await kphp::forks::id_managed([](std::chrono::seconds seconds) noexcept { return kphp::coro::io_scheduler::get().schedule(seconds); }, + std::chrono::seconds{seconds}); co_return; } diff --git a/runtime-light/stdlib/time/timer-functions.h b/runtime-light/stdlib/time/timer-functions.h index e6e0bdbc1f..5efa5250cc 100644 --- a/runtime-light/stdlib/time/timer-functions.h +++ b/runtime-light/stdlib/time/timer-functions.h @@ -24,9 +24,10 @@ void f$set_timer(int64_t timeout_ms, T&& on_timer_callback) noexcept { // TODO choose from: // 1. someone should pop that fork from ForkInstanceState since it will stay there unless we perform f$wait on fork // 2. start timer_task via kphp::coro::io_scheduler::spawn (it won't have distinct fork id) - auto timer_task{std::invoke( + auto timer_task{kphp::coro::detach_task( [](std::chrono::milliseconds duration, T on_timer_callback) noexcept -> kphp::coro::task<> { - co_await kphp::forks::id_managed(kphp::coro::io_scheduler::get().schedule(duration)); + co_await kphp::forks::id_managed([](std::chrono::milliseconds duration) noexcept { return kphp::coro::io_scheduler::get().schedule(duration); }, + duration); std::invoke(std::move(on_timer_callback)); }, std::chrono::milliseconds{timeout_ms}, std::forward(on_timer_callback))}; diff --git a/runtime-light/streams/connection.h b/runtime-light/streams/connection.h index 0a52c083b5..4e44db20f1 100644 --- a/runtime-light/streams/connection.h +++ b/runtime-light/streams/connection.h @@ -12,6 +12,7 @@ #include #include "runtime-common/core/runtime-core.h" +#include "runtime-light/coroutine/detached-task.h" #include "runtime-light/coroutine/event.h" #include "runtime-light/k2-platform/k2-api.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -140,7 +141,8 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s } const auto finalizer{vk::finally([state] noexcept { state.get()->m_unwatch_event.reset(); })}; - const auto v{co_await kphp::coro::when_any(unwatch_awaiter(std::move(state)), descriptor_awaiter(descriptor))}; + const auto v{ + co_await kphp::coro::when_any(kphp::coro::detach_task(unwatch_awaiter, std::move(state)), kphp::coro::detach_task(descriptor_awaiter, descriptor))}; if (std::holds_alternative(v)) { if constexpr (kphp::coro::is_async_function_v) { co_await std::invoke(std::move(h)); @@ -151,7 +153,8 @@ auto connection::register_abort_handler(on_abort_handler_type&& h) noexcept -> s }}; m_shared_state.get()->m_unwatch_event.emplace(); - if (!kphp::coro::io_scheduler::get().spawn(watcher(m_stream.descriptor(), m_shared_state, std::forward(h)))) [[unlikely]] { + if (!kphp::coro::io_scheduler::get().spawn(kphp::coro::detach_task(watcher, m_stream.descriptor(), m_shared_state, std::forward(h)))) + [[unlikely]] { m_shared_state.get()->m_unwatch_event.reset(); return std::unexpected{k2::errno_ebusy}; } diff --git a/tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp b/tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp new file mode 100644 index 0000000000..010a64cf38 --- /dev/null +++ b/tests/cpp/runtime/memory_resource/chunk-pool-resource-test.cpp @@ -0,0 +1,205 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include + +#include + +#include "runtime-common/core/memory-resource/chunk-pool-resource.h" + +namespace { + +constexpr size_t CHUNK_SIZE{64}; +// mirrors chunk_pool_resource's internal list_node layout: a single pointer, no padding +constexpr size_t HEADER_SIZE{sizeof(void*)}; + +template +using aligned_buffer = std::array; + +} // namespace + +TEST(chunk_pool_resource_test, uninited_state) { + memory_resource::chunk_pool_resource resource; + + ASSERT_EQ(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate0(), nullptr); + ASSERT_EQ(resource.get_buffer_list_head(), nullptr); +} + +TEST(chunk_pool_resource_test, single_chunk_exact_buffer) { + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + void* mem{resource.allocate()}; + + ASSERT_EQ(mem, static_cast(buffer.data() + HEADER_SIZE)); + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, multiple_chunks_exact_fit) { + constexpr size_t CHUNKS{5}; + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + for (size_t i = 0; i < CHUNKS; ++i) { + void* mem{resource.allocate()}; + ASSERT_EQ(mem, static_cast(buffer.data() + HEADER_SIZE + i * CHUNK_SIZE)); + } + + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, truncated_tail_is_unused) { + constexpr size_t CHUNKS{3}; + constexpr size_t TAIL{CHUNK_SIZE - 1}; // smaller than one chunk, must be dropped + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + for (size_t i = 0; i < CHUNKS; ++i) { + ASSERT_NE(resource.allocate(), nullptr); + } + + // the trailing tail bytes don't make up a whole chunk, so they must not be handed out + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, minimal_chunk_size) { + constexpr size_t MIN_CHUNK_SIZE{sizeof(void*)}; + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), MIN_CHUNK_SIZE); + + void* a{resource.allocate()}; + void* b{resource.allocate()}; + + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + ASSERT_NE(a, b); + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, allocate0_returns_zeroed_memory) { + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + void* mem{resource.allocate()}; + + ASSERT_NE(mem, nullptr); + + std::memset(mem, 0xFF, CHUNK_SIZE); + resource.deallocate(mem); + void* mem0{resource.allocate0()}; + + ASSERT_EQ(mem0, mem); + + auto* bytes{static_cast(mem0)}; + for (size_t i = 0; i < CHUNK_SIZE; ++i) { + ASSERT_EQ(static_cast(bytes[i]), 0); + } +} + +TEST(chunk_pool_resource_test, allocate0_on_exhausted_pool_returns_null) { + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate0(), nullptr); +} + +TEST(chunk_pool_resource_test, chunks_do_not_overlap) { + constexpr size_t CHUNKS{4}; + alignas(alignof(void*)) aligned_buffer buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer.data(), buffer.size(), CHUNK_SIZE); + + std::array chunks{}; + for (size_t i = 0; i < CHUNKS; ++i) { + chunks[i] = resource.allocate(); + + ASSERT_NE(chunks[i], nullptr); + + std::memset(chunks[i], static_cast(i + 1), CHUNK_SIZE); + } + + for (size_t i = 0; i < CHUNKS; ++i) { + auto* bytes{static_cast(chunks[i])}; + for (size_t j = 0; j < CHUNK_SIZE; ++j) { + ASSERT_EQ(bytes[j], static_cast(i + 1)); + } + } +} + +TEST(chunk_pool_resource_test, add_extra_memory_grows_pool_after_exhaustion) { + alignas(alignof(void*)) aligned_buffer buffer1{}; + alignas(alignof(void*)) aligned_buffer buffer2{}; + memory_resource::chunk_pool_resource resource; + resource.init(buffer1.data(), buffer1.size(), CHUNK_SIZE); + + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate(), nullptr); // exhausted + + resource.add_extra_memory(buffer2.data(), buffer2.size()); + + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_NE(resource.allocate(), nullptr); + ASSERT_EQ(resource.allocate(), nullptr); // exhausted again +} + +TEST(chunk_pool_resource_test, add_extra_memory_preserves_existing_free_chunks) { + constexpr size_t OLD_CHUNKS{2}; + alignas(alignof(void*)) aligned_buffer old_buffer{}; + alignas(alignof(void*)) aligned_buffer extra_buffer{}; + memory_resource::chunk_pool_resource resource; + resource.init(old_buffer.data(), old_buffer.size(), CHUNK_SIZE); + + // consume the first chunk of old_buffer, the second one is left in the free list + ASSERT_NE(resource.allocate(), nullptr); + + void* remaining_old_chunk{static_cast(old_buffer.data() + HEADER_SIZE + CHUNK_SIZE)}; + resource.add_extra_memory(extra_buffer.data(), extra_buffer.size()); + + // the newly added buffer's chunk is handed out before the old buffer's leftover chunk + ASSERT_EQ(resource.allocate(), static_cast(extra_buffer.data() + HEADER_SIZE)); + ASSERT_EQ(resource.allocate(), remaining_old_chunk); + ASSERT_EQ(resource.allocate(), nullptr); +} + +TEST(chunk_pool_resource_test, buffer_list_head_tracks_most_recently_added_buffer) { + alignas(alignof(void*)) aligned_buffer buffer1{}; + alignas(alignof(void*)) aligned_buffer buffer2{}; + alignas(alignof(void*)) aligned_buffer buffer3{}; + + memory_resource::chunk_pool_resource resource; + + resource.init(buffer1.data(), buffer1.size(), CHUNK_SIZE); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer1.data())); + + resource.add_extra_memory(buffer2.data(), buffer2.size()); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer2.data())); + + resource.add_extra_memory(buffer3.data(), buffer3.size()); + + ASSERT_EQ(static_cast(resource.get_buffer_list_head()), static_cast(buffer3.data())); + + auto* head{resource.get_buffer_list_head()}; + + ASSERT_EQ(static_cast(head->next), static_cast(buffer2.data())); + + head = head->next; + + ASSERT_EQ(static_cast(head->next), static_cast(buffer1.data())); + + head = head->next; + + ASSERT_EQ(static_cast(head->next), nullptr); +} diff --git a/tests/cpp/runtime/memory_resource/stack-resource-test.cpp b/tests/cpp/runtime/memory_resource/stack-resource-test.cpp new file mode 100644 index 0000000000..18e00b7be8 --- /dev/null +++ b/tests/cpp/runtime/memory_resource/stack-resource-test.cpp @@ -0,0 +1,170 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2026 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include +#include +#include + +#include + +#include "runtime-common/core/memory-resource/stack-resource.h" + +namespace { + +constexpr size_t BUFFER_SIZE{64}; + +} // namespace + +TEST(stack_resource_test, uninited_state) { + memory_resource::stack_resource resource; + + ASSERT_EQ(resource.memory_begin(), nullptr); +} + +TEST(stack_resource_test, single_allocation_exact_buffer_fit) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* mem{resource.allocate(BUFFER_SIZE)}; + + ASSERT_EQ(mem, static_cast(buffer.data())); + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(stack_resource_test, sequential_allocations_bump_pointer) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* a{resource.allocate(16)}; + void* b{resource.allocate(16)}; + void* c{resource.allocate(32)}; + + ASSERT_EQ(a, static_cast(buffer.data())); + ASSERT_EQ(b, static_cast(buffer.data() + 16)); + ASSERT_EQ(c, static_cast(buffer.data() + 32)); + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(stack_resource_test, allocation_exceeding_remaining_capacity_returns_null_without_corrupting_state) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* mem{resource.allocate(BUFFER_SIZE - 16)}; + + ASSERT_NE(mem, nullptr); + ASSERT_EQ(resource.allocate(32), nullptr); + + void* rest{resource.allocate(16)}; + + ASSERT_EQ(rest, static_cast(buffer.data() + (BUFFER_SIZE - 16))); + ASSERT_EQ(resource.allocate(1), nullptr); +} + +TEST(stack_resource_test, lifo_deallocate_reuses_address) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + resource.allocate(16); + void* b1{resource.allocate(16)}; + + resource.deallocate(b1, 16); + void* b2{resource.allocate(16)}; + + ASSERT_EQ(b2, b1); +} + +TEST(stack_resource_test, multiple_lifo_deallocations_restore_cursor_in_order) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* a{resource.allocate(16)}; + void* b{resource.allocate(16)}; + void* c{resource.allocate(16)}; + + resource.deallocate(c, 16); + resource.deallocate(b, 16); + + void* b_again{resource.allocate(16)}; + void* c_again{resource.allocate(16)}; + + ASSERT_EQ(b_again, b); + ASSERT_EQ(c_again, c); + + resource.deallocate(c_again, 16); + resource.deallocate(b_again, 16); + resource.deallocate(a, 16); + + ASSERT_EQ(resource.allocate(BUFFER_SIZE), static_cast(buffer.data())); +} + +TEST(stack_resource_test, allocate0_returns_zeroed_memory) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* mem{resource.allocate(BUFFER_SIZE)}; + ASSERT_NE(mem, nullptr); + std::memset(mem, 0xAB, BUFFER_SIZE); + resource.deallocate(mem, BUFFER_SIZE); + + void* mem0{resource.allocate0(BUFFER_SIZE)}; + + ASSERT_EQ(mem0, mem); + + auto* bytes{static_cast(mem0)}; + for (size_t i = 0; i < BUFFER_SIZE; ++i) { + ASSERT_EQ(static_cast(bytes[i]), 0); + } +} + +TEST(stack_resource_test, allocate0_on_full_buffer_returns_null) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + ASSERT_NE(resource.allocate(BUFFER_SIZE), nullptr); + ASSERT_EQ(resource.allocate0(1), nullptr); +} + +TEST(stack_resource_test, allocations_do_not_overlap) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + void* a{resource.allocate(20)}; + void* b{resource.allocate(20)}; + void* c{resource.allocate(20)}; + + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + ASSERT_NE(c, nullptr); + + std::memset(a, 1, 20); + std::memset(b, 2, 20); + std::memset(c, 2, 20); // intentionally same fill as b, to catch an overlap via pointer identity instead + + ASSERT_NE(a, b); + ASSERT_NE(b, c); + ASSERT_EQ(static_cast(b) - static_cast(a), 20); + ASSERT_EQ(static_cast(c) - static_cast(b), 20); +} + +TEST(stack_resource_test, memory_begin_returns_initial_buffer_and_is_stable) { + std::vector buffer(BUFFER_SIZE); + memory_resource::stack_resource resource; + resource.init(buffer.data(), buffer.size()); + + ASSERT_EQ(resource.memory_begin(), static_cast(buffer.data())); + + void* mem{resource.allocate(16)}; + ASSERT_EQ(resource.memory_begin(), static_cast(buffer.data())); + + resource.deallocate(mem, 16); + ASSERT_EQ(resource.memory_begin(), static_cast(buffer.data())); +} diff --git a/tests/cpp/runtime/runtime-tests.cmake b/tests/cpp/runtime/runtime-tests.cmake index 731da98fe6..e1ec8124fd 100644 --- a/tests/cpp/runtime/runtime-tests.cmake +++ b/tests/cpp/runtime/runtime-tests.cmake @@ -14,10 +14,12 @@ prepend(RUNTIME_TESTS_SOURCES ${BASE_DIR}/tests/cpp/runtime/ number-string-comparison.cpp kphp-type-traits-test.cpp msgpack-test.cpp + memory_resource/chunk-pool-resource-test.cpp memory_resource/details/memory_chunk_list-test.cpp memory_resource/details/memory_chunk_tree-test.cpp memory_resource/details/memory_ordered_chunk_list-test.cpp memory_resource/extra-memory-pool-test.cpp + memory_resource/stack-resource-test.cpp memory_resource/unsynchronized_pool_resource-test.cpp string-list-test.cpp string-test.cpp