From 517f773a0ecab1978f0f1c433cfe38f201967a21 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 2 Jul 2026 18:14:11 +0300 Subject: [PATCH 01/30] audio: pipeline: enable position reporting for user-space pipelines Place the pipeline position lookup table in the sysuser memory partition and replace k_spinlock with a dynamically allocated k_mutex when CONFIG_SOF_USERSPACE_LL is enabled. Spinlocks disable interrupts which is a privileged operation unavailable from user-mode threads. The mutex pointer is stored in a separate APP_SYSUSER_BSS variable outside the SHARED_DATA struct so Zephyr's kernel object tracking can recognize it for syscall verification. Move pipeline_posn_init() from task_main_start() to primary_core_init() before platform_init(), so the mutex is allocated before ipc_user_init() grants thread access to it. In pipeline_posn_get(), bypass the sof_get() kernel singleton and access the shared structure directly when running in user-space. Grant the ipc_user_init thread access to the pipeline position mutex via new pipeline_posn_grant_access() helper. Signed-off-by: Kai Vehmanen --- src/audio/pipeline/pipeline-graph.c | 90 +++++++++++++++++++++++--- src/include/sof/audio/pipeline-trace.h | 4 ++ src/include/sof/audio/pipeline.h | 8 +++ src/init/init.c | 6 ++ src/ipc/ipc-common.c | 1 + zephyr/wrapper.c | 3 - 6 files changed, 99 insertions(+), 13 deletions(-) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 6e154dfbfba6..cb8b3860cecd 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -44,10 +45,20 @@ DECLARE_TR_CTX(pipe_tr, SOF_UUID(pipe_uuid), LOG_LEVEL_INFO); /* lookup table to determine busy/free pipeline metadata objects */ struct pipeline_posn { bool posn_offset[PPL_POSN_OFFSETS]; /**< available offsets */ +#ifndef CONFIG_SOF_USERSPACE_LL struct k_spinlock lock; /**< lock mechanism */ +#endif }; /* the pipeline position lookup table */ -static SHARED_DATA struct pipeline_posn pipeline_posn_shared; +static APP_SYSUSER_BSS SHARED_DATA struct pipeline_posn pipeline_posn_shared; + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Mutex pointer in user-accessible partition so user-space threads + * can read the pointer for syscalls. Kept outside the SHARED_DATA + * struct to avoid kernel object tracking issues. + */ +static APP_SYSUSER_BSS struct k_mutex *pipeline_posn_mutex; +#endif /** * \brief Retrieves pipeline position structure. @@ -55,9 +66,48 @@ static SHARED_DATA struct pipeline_posn pipeline_posn_shared; */ static inline struct pipeline_posn *pipeline_posn_get(void) { +#ifdef CONFIG_SOF_USERSPACE_LL + return &pipeline_posn_shared; +#else return sof_get()->pipeline_posn; +#endif +} + +/* + * Position table locking. User-space LL cannot use a spinlock (disabling + * interrupts is privileged), so it uses a mutex; the config split is kept + * here so the callers below stay identical for both configurations. + */ +#ifdef CONFIG_SOF_USERSPACE_LL +typedef int pipeline_posn_key_t; + +static inline pipeline_posn_key_t pipeline_posn_lock(struct pipeline_posn *posn) +{ + (void)posn; + k_mutex_lock(pipeline_posn_mutex, K_FOREVER); + return 0; } +static inline void pipeline_posn_unlock(struct pipeline_posn *posn, pipeline_posn_key_t key) +{ + (void)posn; + (void)key; + k_mutex_unlock(pipeline_posn_mutex); +} +#else +typedef k_spinlock_key_t pipeline_posn_key_t; + +static inline pipeline_posn_key_t pipeline_posn_lock(struct pipeline_posn *posn) +{ + return k_spin_lock(&posn->lock); +} + +static inline void pipeline_posn_unlock(struct pipeline_posn *posn, pipeline_posn_key_t key) +{ + k_spin_unlock(&posn->lock, key); +} +#endif + /** * \brief Retrieves first free pipeline position offset. * \param[in,out] posn_offset Pipeline position offset to be set. @@ -68,9 +118,7 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int ret = -EINVAL; uint32_t i; - k_spinlock_key_t key; - - key = k_spin_lock(&pipeline_posn->lock); + pipeline_posn_key_t key = pipeline_posn_lock(pipeline_posn); for (i = 0; i < PPL_POSN_OFFSETS; ++i) { if (!pipeline_posn->posn_offset[i]) { @@ -81,8 +129,7 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) } } - - k_spin_unlock(&pipeline_posn->lock, key); + pipeline_posn_unlock(pipeline_posn, key); return ret; } @@ -95,20 +142,34 @@ static inline void pipeline_posn_offset_put(uint32_t posn_offset) { struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int i = posn_offset / sizeof(struct sof_ipc_stream_posn); - k_spinlock_key_t key; - - key = k_spin_lock(&pipeline_posn->lock); + pipeline_posn_key_t key = pipeline_posn_lock(pipeline_posn); pipeline_posn->posn_offset[i] = false; - k_spin_unlock(&pipeline_posn->lock, key); + pipeline_posn_unlock(pipeline_posn, key); } void pipeline_posn_init(struct sof *sof) { sof->pipeline_posn = &pipeline_posn_shared; +#ifdef CONFIG_SOF_USERSPACE_LL + pipeline_posn_mutex = k_object_alloc(K_OBJ_MUTEX); + if (!pipeline_posn_mutex) { + pipe_cl_err("pipeline posn mutex alloc failed"); + k_panic(); + } + k_mutex_init(pipeline_posn_mutex); +#else k_spinlock_init(&sof->pipeline_posn->lock); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +void pipeline_posn_grant_access(struct k_thread *thread) +{ + k_thread_access_grant(thread, pipeline_posn_mutex); } +#endif /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority, @@ -140,12 +201,21 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ p->pipeline_id = pipeline_id; p->status = COMP_STATE_INIT; p->trigger.cmd = COMP_TRIGGER_NO_ACTION; + +#ifndef CONFIG_SOF_USERSPACE_LL + /* + * pipe_tr lives in the .trace_ctx section, which is not mapped into + * the sysuser partition, so it cannot be read from a user-mode thread. + * The copy is also unnecessary in that configuration: with Zephyr + * logging the pipe_*() macros use the global pipe_tr, not p->tctx. + */ ret = memcpy_s(&p->tctx, sizeof(struct tr_ctx), &pipe_tr, sizeof(struct tr_ctx)); if (ret < 0) { pipe_err(p, "failed to copy trace settings"); goto free; } +#endif ret = pipeline_posn_offset_get(&p->posn_offset); if (ret < 0) { diff --git a/src/include/sof/audio/pipeline-trace.h b/src/include/sof/audio/pipeline-trace.h index d1f89583ad45..dbd5ae7f7c84 100644 --- a/src/include/sof/audio/pipeline-trace.h +++ b/src/include/sof/audio/pipeline-trace.h @@ -59,6 +59,10 @@ extern struct tr_ctx pipe_tr; #else +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_USERSPACE_LL) +#error "Invalid build config: User-space cannot access trace context." +#endif + #define pipe_err(pipe_p, __e, ...) \ trace_dev_err(trace_pipe_get_tr_ctx, trace_pipe_get_id, \ trace_pipe_get_subid, pipe_p, __e, ##__VA_ARGS__) diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 913a569c208c..ff456fbceb7d 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -206,6 +206,14 @@ int pipeline_complete(struct pipeline *p, struct comp_dev *source, */ void pipeline_posn_init(struct sof *sof); +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * \brief Grants user-space thread access to pipeline position mutex. + * \param[in] thread Thread to grant access to. + */ +void pipeline_posn_grant_access(struct k_thread *thread); +#endif + /** * \brief Resets the pipeline and free runtime resources. * \param[in] p pipeline. diff --git a/src/init/init.c b/src/init/init.c index 7976e2eb673e..5990cfebc2dc 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #if CONFIG_IPC_MAJOR_4 #include @@ -232,6 +233,11 @@ __cold static int primary_core_init(int argc, char *argv[], struct sof *sof) zephyr_ll_user_resources_init(); #endif + /* init pipeline position offsets - must be before platform_init() + * which calls ipc_init() -> ipc_user_init() that needs the posn mutex. + */ + pipeline_posn_init(sof); + /* init the platform */ if (platform_init(sof) < 0) sof_panic(SOF_IPC_PANIC_PLATFORM); diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 96b957f9fabb..c5e2727ab3cb 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -457,6 +457,7 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) user_grant_dma_access_all(ipc_user->thread); k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); + pipeline_posn_grant_access(ipc_user->thread); return 0; diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index 9bbb43f8a798..afdc5b54a2e9 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -177,9 +177,6 @@ int task_main_start(struct sof *sof) /* init default audio components */ sys_comp_init(sof); - /* init pipeline position offsets */ - pipeline_posn_init(sof); - return 0; } From 60a91d6a9097b442a51df12213bbcbfac8def7e3 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 13 Aug 2026 19:42:30 +0300 Subject: [PATCH 02/30] boards: intel: default to user-space LL for ptl and wcl Make the options from app/overlays/ptl/ll_userspace_overlay.conf the default for the Intel Panther Lake (ptl) and Wildcat Lake (wcl) build targets, so user-space Low-Latency audio pipelines are enabled without having to pass the overlay explicitly. As noted in the overlay header, once user-space LL is enabled for a target by default the settings belong in the SOF board file directly. For ptl the board already provides the user-space base (USERSPACE, dynamic threads, MMU L2 tables, domain partitions), so only the LL overlay options are added and the conflicting telemetry / cold-store / llext / modules defaults are flipped to match the overlay. wcl had no user-space base at all; since CONFIG_SOF_USERSPACE_LL depends on CONFIG_USERSPACE it would otherwise be silently dropped. Mirror ptl's user-space base into the wcl board file as well so LL actually takes effect there. The ll_userspace_overlay.conf file is kept unchanged; it now re-applies identical values and remains usable by development build scripts. Signed-off-by: Kai Vehmanen --- app/boards/intel_adsp_ace30_ptl.conf | 28 +++++++++++++++---- app/boards/intel_adsp_ace30_wcl.conf | 41 ++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index b6ac41938398..9a63751b1bde 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y -CONFIG_COLD_STORE_EXECUTE_DRAM=y +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n +CONFIG_COLD_STORE_EXECUTE_DRAM=n # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -40,10 +40,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=y -CONFIG_LLEXT_EXPERIMENTAL=y +CONFIG_LLEXT_STORAGE_WRITABLE=n +CONFIG_LLEXT_EXPERIMENTAL=n CONFIG_LLEXT_EDK=n -CONFIG_MODULES=y +CONFIG_MODULES=n # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -78,3 +78,19 @@ CONFIG_SOF_USERSPACE_PROXY=y CONFIG_MAX_THREAD_BYTES=3 CONFIG_MAX_DOMAIN_PARTITIONS=32 + +# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) +# Run Low-Latency audio pipelines in user-space threads by default. +CONFIG_SOF_USERSPACE_LL=y +CONFIG_SOF_USERSPACE_INTERFACE_DMA=y +CONFIG_DAI_USERSPACE=y + +# Settings currently required to enable user-space LL. The cold-store, +# telemetry, loadable-module and misc feature disables above/here are not +# yet user-space compatible (see the former overlay for rationale). +CONFIG_COLD_STORE_EXECUTE_DEBUG=n +CONFIG_SOF_BOOT_TEST_ALLOWED=n +CONFIG_CROSS_CORE_STREAM=n +CONFIG_INTEL_ADSP_MIC_PRIVACY=n +CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index 2196af333e65..d825a2a37c95 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y -CONFIG_COLD_STORE_EXECUTE_DRAM=y +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n +CONFIG_COLD_STORE_EXECUTE_DRAM=n # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -39,10 +39,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=y -CONFIG_LLEXT_EXPERIMENTAL=y +CONFIG_LLEXT_STORAGE_WRITABLE=n +CONFIG_LLEXT_EXPERIMENTAL=n CONFIG_LLEXT_EDK=n -CONFIG_MODULES=y +CONFIG_MODULES=n # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -64,3 +64,32 @@ CONFIG_PM_DEVICE_RUNTIME_ASYNC=n CONFIG_LOG_BACKEND_ADSP=n CONFIG_LOG_FLUSH_SLEEP_US=5000 CONFIG_WINSTREAM_CONSOLE=n + +# Userspace base (mirrored from intel_adsp_ace30_ptl.conf) +# Required so that user-space LL (below) can actually be enabled, since +# CONFIG_SOF_USERSPACE_LL depends on CONFIG_USERSPACE. +CONFIG_USERSPACE=y +CONFIG_DYNAMIC_THREAD=y +CONFIG_DYNAMIC_THREAD_ALLOC=y +CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y +CONFIG_SOF_STACK_SIZE=8192 +CONFIG_SOF_USERSPACE_PROXY=y +CONFIG_MAX_THREAD_BYTES=3 +CONFIG_MAX_DOMAIN_PARTITIONS=32 +CONFIG_XTENSA_MMU_NUM_L2_TABLES=128 + +# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) +# Run Low-Latency audio pipelines in user-space threads by default. +CONFIG_SOF_USERSPACE_LL=y +CONFIG_SOF_USERSPACE_INTERFACE_DMA=y +CONFIG_DAI_USERSPACE=y + +# Settings currently required to enable user-space LL. The cold-store, +# telemetry, loadable-module and misc feature disables above/here are not +# yet user-space compatible (see the former overlay for rationale). +CONFIG_COLD_STORE_EXECUTE_DEBUG=n +CONFIG_SOF_BOOT_TEST_ALLOWED=n +CONFIG_CROSS_CORE_STREAM=n +CONFIG_INTEL_ADSP_MIC_PRIVACY=n +CONFIG_XRUN_NOTIFICATIONS_ENABLE=n +CONFIG_ZEPHYR_DP_SCHEDULER=n From 7216faa4fe65e86446d29fa3fac110e207f7c494 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 21 May 2026 14:13:46 +0200 Subject: [PATCH 03/30] ipc: only add cold partitions if they're non-empty .cold and .coldrodata partitions can be empty, avoid a failure in such cases. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-common.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index c5e2727ab3cb..0e7b344510cd 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -490,6 +490,44 @@ __cold static void ipc_user_init(void) if (ret < 0) LOG_WRN("ipc context partition add failed: %d", ret); + /* + * Grant user-space access to .cold (execute) and .coldrodata (read) + * sections in IMR. The prepare path walks component code that may + * reference __cold functions and __cold_rodata data. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __cold_start[], __cold_end[]; + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&__cold_end[0] > &__cold_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__cold_start, + .size = ALIGN_UP((uintptr_t)__cold_end - (uintptr_t)__cold_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RX_U_RX, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold text partition add failed: %d", ret); + } + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif + k_sem_init(ipc_user->sem, 0, 1); ret = ipc_user_init_thread(ipc_user); From 5f219e6429009470993445720e53480d0c02b681 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 21 May 2026 16:55:15 +0200 Subject: [PATCH 04/30] audio: pipeline: add a missing header Add a missing header for the zephyr_ll_(un)lock_sched() functions. Signed-off-by: Guennadi Liakhovetski --- src/audio/pipeline/pipeline-graph.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index cb8b3860cecd..816b94533203 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include From 5f22a2f0e613be43e5530900d9f95a0bdb4b5a7f Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 10:53:50 +0200 Subject: [PATCH 05/30] lib-manager: add a syscall to handle LLEXT-related work Extract a privileged LLEXT-related part from lib_manager_module_create() into a separate function to be called from kernel context. At the same time lib_manager_mod_free_priv() already executes privileged operations; to make it callable in userspace, convert lib_manager_free_module() to a system call. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 19 +++++ src/library_manager/lib_manager.c | 111 +++++++++++++++++++++--------- zephyr/CMakeLists.txt | 1 + 3 files changed, 98 insertions(+), 33 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index fb277ac75f64..d52f8e047b79 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -219,6 +219,25 @@ void lib_manager_get_instance_bss_address(uint32_t instance_id, */ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type); +struct userspace_context; +/* + * \brief Allocate the module and start the agent if needed + */ +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops); + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall int lib_manager_free_module(const uint32_t component_id); + +#include +#else +int z_impl_lib_manager_free_module(const uint32_t component_id); +#define lib_manager_free_module z_impl_lib_manager_free_module +#endif + /* * \brief Initialize message * diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 1b1dd5f2cc9e..35f442f47fec 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -415,7 +415,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const * * Function is responsible to free module resources in HP memory. */ -static int lib_manager_free_module(const uint32_t component_id) +int z_impl_lib_manager_free_module(const uint32_t component_id) { const struct sof_man_module *mod; const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -462,7 +462,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const return 0; } -static int lib_manager_free_module(const uint32_t component_id) +static int z_impl_lib_manager_free_module(const uint32_t component_id) { /* Since we cannot allocate the freeing is not considered to be an error */ tr_warn(&lib_manager_tr, "Dynamic module freeing is not supported"); @@ -642,34 +642,36 @@ static enum buildinfo_mod_type lib_manager_get_module_type(const struct sof_man_ } } -/* - * \brief Load module code, allocate its instance and create a module adapter component. - * \param[in] drv - component driver pointer. - * \param[in] config - component ipc descriptor pointer. - * \param[in] spec - passdowned data from driver. - * - * \return: a pointer to newly created module adapter component on success. NULL on error. - */ -static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, - const struct comp_ipc_config *config, - const void *spec) +/* Error path resource freeing */ +static void lib_manager_mod_free_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + struct userspace_context *userspace) +{ +#if CONFIG_SOF_USERSPACE_PROXY + if (userspace) + userspace_proxy_destroy(drv, userspace); +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + lib_manager_free_module(config->id); +} + +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops) { const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); - struct userspace_context *userspace = NULL; - const struct module_interface *ops; const struct sof_man_module *mod; system_agent_start_fn agent; - void *adapter_priv = NULL; const void **agent_iface; - struct comp_dev *dev; int ret; #ifdef CONFIG_SOF_USERSPACE_PROXY if (drv->user_heap && config->proc_domain != COMP_PROCESSING_DOMAIN_DP) { tr_err(&lib_manager_tr, "Userspace supports only DP modules."); - return NULL; + return -EOPNOTSUPP; } #endif @@ -677,12 +679,12 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!desc) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", config->id); - return NULL; + return -ENOENT; } if (entry_index >= desc->header.num_module_entries) { tr_err(&lib_manager_tr, "Entry index %u out of bounds.", entry_index); - return NULL; + return -EINVAL; } mod = (const struct sof_man_module *) @@ -693,53 +695,96 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!module_entry_point) { tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + return -ENOENT; } switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: agent = NULL; - ops = (const struct module_interface *)module_entry_point; + *ops = (const struct module_interface *)module_entry_point; agent_iface = NULL; break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; - agent_iface = (const void **)&ops; + agent_iface = (const void **)ops; break; #if CONFIG_INTEL_MODULES case MOD_TYPE_IADK: agent = &system_agent_start; - ops = &processing_module_adapter_interface; - agent_iface = (const void **)&adapter_priv; + *ops = &processing_module_adapter_interface; + agent_iface = (const void **)adapter_priv; break; #endif case MOD_TYPE_INVALID: + default: + ret = -EINVAL; goto err; } if (agent || IS_ENABLED(CONFIG_SOF_USERSPACE_PROXY)) { /* At this point module resources are allocated and it is moved to L2 memory. */ ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, - agent_iface, &userspace, &ops); + agent_iface, userspace, ops); if (ret) goto err; } - if (comp_set_adapter_ops(drv, ops) < 0) + ret = comp_set_adapter_ops(drv, *ops); + if (ret < 0) goto err; - dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, NULL); + return 0; + +err: + lib_manager_mod_free_priv(drv, config, *userspace); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +static int z_vrfy_lib_manager_free_module(const uint32_t component_id) +{ + return z_impl_lib_manager_free_module(component_id); +} +#include + +#endif /* CONFIG_USERSPACE */ + +/* + * \brief Load module code, allocate its instance and create a module adapter component. + * \param[in] drv - component driver pointer. + * \param[in] config - component ipc descriptor pointer. + * \param[in] spec - passdowned data from driver. + * + * \return: a pointer to newly created module adapter component on success. NULL on error. + */ +static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec) +{ + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + void *adapter_priv = NULL; + struct comp_dev *dev; + + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP || + !IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { + int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, + &userspace, &ops); + + if (ret < 0) + return NULL; + } + + dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, ops); if (!dev) goto err; return dev; err: -#if CONFIG_SOF_USERSPACE_PROXY - if (userspace) - userspace_proxy_destroy(drv, userspace); -#endif /* CONFIG_SOF_USERSPACE_PROXY */ - lib_manager_free_module(config->id); + lib_manager_mod_free_priv(drv, config, userspace); return NULL; } diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index e0e7e8bfb302..f09d94524573 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -630,6 +630,7 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) From cbbeb413e8dd0b2a2b4ba71d83e24570663aea16 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 18 Aug 2026 11:35:08 +0200 Subject: [PATCH 06/30] ipc: ipc4: remove a memcpy() in MOD_INIT_INSTANCE Use a type-cast instead of a memcpy() to obtain struct ipc4_module_init_instance data. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 5888b007ed42..b5fbda6a2f9a 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1513,28 +1513,25 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_INIT_INSTANCE: #ifdef CONFIG_SOF_USERSPACE_LL { + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + /* User-space init: kernel does driver lookup only (requires * access to IMR manifest and driver list in kernel memory). * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. * Cross-core creation stays fully in kernel. */ - struct ipc4_module_init_instance mi; - - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - ret = memcpy_s(&mi, sizeof(mi), ipc4, sizeof(*ipc4)); - if (ret < 0) - break; + const struct ipc4_module_init_instance *mi = + (const struct ipc4_module_init_instance *)ipc4; - if (!cpu_is_me(mi.extension.r.core_id)) { + if (!cpu_is_me(mi->extension.r.core_id)) { ret = ipc4_init_module_instance(ipc4); } else { struct ipc *ipc = ipc_get(); - uint32_t comp_id = IPC4_COMP_ID(mi.primary.r.module_id, - mi.primary.r.instance_id); + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); const struct comp_driver *drv = ipc4_get_comp_drv( IPC4_MOD_ID(comp_id)); struct ipc_user *pdata = ipc->ipc_user_pdata; From 9841764be021b10c17871a0edcae6ecd722c846c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 18 Aug 2026 14:28:04 +0200 Subject: [PATCH 07/30] schedule: add a function for finding userspace scherulers scheduler_get_data() only finds scheduler data for kernel mode schedulers. Add a similar function for userspace schedulers. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/schedule.h | 28 +++++++++++++++++++++------- src/schedule/zephyr_ll.c | 10 +++++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index a68169736ef2..530bb83de124 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -186,14 +186,8 @@ struct schedulers **arch_user_schedulers_get(void); struct schedulers **arch_user_schedulers_get_for_core(int core); -/** - * Retrieves scheduler's data. - * @param type SOF_SCHEDULE_ type. - * @return Pointer to scheduler's data. - */ -static inline void *scheduler_get_data(uint16_t type) +static inline void *scheduler_list_get_data(struct schedulers *schedulers, uint16_t type) { - struct schedulers *schedulers = *arch_schedulers_get(); struct schedule_data *sch; struct list_item *slist; @@ -209,6 +203,26 @@ static inline void *scheduler_get_data(uint16_t type) return NULL; } +/** + * Retrieves scheduler's data. + * @param type SOF_SCHEDULE_ type. + * @return Pointer to scheduler's data. + */ +static inline void *scheduler_get_data(uint16_t type) +{ + return scheduler_list_get_data(*arch_schedulers_get(), type); +} + +/** + * Retrieves userspace scheduler's data. + * @param type SOF_SCHEDULE_ type. + * @return Pointer to scheduler's data. + */ +static inline void *scheduler_get_user_data(uint16_t type) +{ + return scheduler_list_get_data(*arch_user_schedulers_get(), type); +} + /** See scheduler_ops::schedule_task_running */ static inline int schedule_task_running(struct task *task) { diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 145416afad3e..61e852fb9c92 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -889,9 +889,13 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { uint32_t flags; +#if CONFIG_SOF_USERSPACE_LL + struct zephyr_ll *ll_sch = scheduler_get_user_data(SOF_SCHEDULE_LL_TIMER); +#else + struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_LL; - struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); zephyr_ll_lock(ll_sch, &flags); scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); @@ -901,7 +905,11 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, /* Return a pointer to the LL scheduler timer domain */ struct ll_schedule_domain *zephyr_ll_domain(void) { +#if CONFIG_SOF_USERSPACE_LL + struct zephyr_ll *ll_sch = scheduler_get_user_data(SOF_SCHEDULE_LL_TIMER); +#else struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif return ll_sch ? ll_sch->ll_domain : NULL; } From a40b80f06ec55c8bdba86becb06b9c483a89b64d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 12:54:31 +0200 Subject: [PATCH 08/30] ipc: add the DP case to module initialization ipc4_init_module_instance() should be called when CONFIG_SOF_USERSPACE_LL isn't selected but also when initializing a DP module. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 39 +++++++++++++++++-------------------- src/ipc/ipc4/helper.c | 28 +++++++++++--------------- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index b5fbda6a2f9a..13954d7929e9 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1501,6 +1501,7 @@ __cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + const struct ipc4_module_init_instance *mi; uint32_t type; int ret; @@ -1511,24 +1512,21 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - * Cross-core creation stays fully in kernel. - */ - const struct ipc4_module_init_instance *mi = - (const struct ipc4_module_init_instance *)ipc4; + mi = (const struct ipc4_module_init_instance *)ipc4; - if (!cpu_is_me(mi->extension.r.core_id)) { - ret = ipc4_init_module_instance(ipc4); - } else { + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && + IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { +#ifdef CONFIG_SOF_USERSPACE_LL + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + * Cross-core creation stays fully in kernel. + */ struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); @@ -1561,11 +1559,10 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, pdata->init_drv = drv; ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); - } - } -#else - ret = ipc4_init_module_instance(ipc4); #endif + } else { + ret = ipc4_init_module_instance(ipc4); + } break; case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index f1e112fb37ea..476fbf438d8b 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -177,18 +177,16 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (!drv) return NULL; -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init->extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else + if (!module_init->extension.r.proc_domain) { ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else /* CONFIG_ZEPHYR_DP_SCHEDULER */ - if (module_init->extension.r.proc_domain) { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { + tr_err(&ipc_tr, + "ipc: DP scheduling is disabled, cannot create comp 0x%x", + comp_id); return NULL; } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { @@ -288,20 +286,16 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, #endif data = ipc4_get_comp_new_data(); -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init.extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else + if (!module_init.extension.r.proc_domain) { ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else - if (module_init.extension.r.proc_domain) { + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); return NULL; } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { From c90125f67caa9b399bdc1b8ccc8f0deead27d0db Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 13:52:52 +0200 Subject: [PATCH 09/30] llext: with userspace let LL thread access DP modules too If LL runs in userspace, it needs access to loaded LLEXT modules, running in DP more too. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 4ad3c55d55bc..d1f6740d1eaf 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -302,12 +302,10 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) mctx->mapped = true; #ifdef CONFIG_SOF_USERSPACE_LL - if (!mctx->domain_dp) { - ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); - if (ret < 0) { - tr_err(&lib_manager_tr, "failed to add domain: %d", ret); - goto e_data; - } + ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); + if (ret < 0) { + tr_err(&lib_manager_tr, "failed to add domain: %d", ret); + goto e_data; } #endif @@ -377,8 +375,7 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->mapped = false; #ifdef CONFIG_SOF_USERSPACE_LL - if (!mctx->domain_dp) - llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); + llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); #endif return err; From aa3487af892d7afd32fe82f305053a49e918d25c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 6 Jul 2026 16:58:35 +0200 Subject: [PATCH 10/30] audio: module-adapter: add two system calls Add two syscall functions to allocate and map, and to unmap vregion for userspace modules. For now only used for DP modules. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/comp_buffer.c | 4 +- src/audio/module_adapter/module_adapter.c | 102 ++++++++++++++++-- .../sof/audio/module_adapter/module/generic.h | 9 ++ zephyr/include/rtos/alloc.h | 2 + 4 files changed, 107 insertions(+), 10 deletions(-) diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8a3d44133d4b..f64b74add169 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); - if (!vregion_put(alloc->vreg)) - rfree(alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index a2b293e4ad0d..a7499bb87498 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -58,14 +58,80 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, #define PAGE_SZ HOST_PAGE_SIZE #endif -static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) +struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ /* FIXME: the size will be derived from configuration */ const size_t buf_size = 28 * 1024; + struct vregion *vr = vregion_create(buf_size); - return vregion_create(buf_size); + if (!vr) + return NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + vregion_mem_info(vr, vreg_size, vreg_start); + + /* + * In the userspace LL case allocations are also performed by the + * userspace IPC thread, which is also the one, executing this syscall + */ + struct k_mem_partition part = { + .start = *vreg_start, + .size = *vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part); + + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part); + if (ret < 0) { + vregion_put(vr); + return NULL; + } +#else + ARG_UNUSED(vreg_start); + ARG_UNUSED(vreg_size); +#endif + + return vr; +} + +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + struct k_mem_partition part = { + .start = alloc->vreg_start, + .size = alloc->vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); +#else + ARG_UNUSED(alloc); +#endif +} + +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc) +{ + if (vregion_put(alloc->vreg)) + return; + + module_adapter_vreg_unmap(alloc); + + sof_heap_free(alloc->heap, alloc); } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, @@ -84,11 +150,12 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - size_t heap_size; + size_t vreg_size; + uintptr_t vreg_start; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + mod_vreg = module_adapter_vreg_new(config, &vreg_start, &vreg_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; @@ -105,7 +172,6 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv #else mod_heap = drv->user_heap; #endif - heap_size = 0; mod_vreg = NULL; } @@ -129,6 +195,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv memset(mod, 0, sizeof(*mod)); alloc->heap = mod_heap; alloc->vreg = mod_vreg; + alloc->vreg_start = vreg_start; + alloc->vreg_size = vreg_size; mod->priv.resources.alloc = alloc; mod_resource_init(mod); @@ -169,6 +237,25 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv return NULL; } +#ifdef CONFIG_USERSPACE +#include +struct vregion *z_vrfy_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size))); + K_OOPS(K_SYSCALL_MEMORY_READ(config, sizeof(*config))); + return z_impl_module_adapter_vreg_new(config, vreg_start, vreg_size); +} +#include +void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ + K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc))); + z_impl_module_adapter_vreg_unmap(alloc); +} +#include +#endif + static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; @@ -186,8 +273,7 @@ static void module_adapter_mem_free(struct processing_module *mod) vregion_free(mod_vreg, mod->dev); vregion_free(mod_vreg, mod); - if (!vregion_put(mod_vreg)) - sof_heap_free(alloc->heap, alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(mod_heap, mod->dev); sof_heap_free(mod_heap, mod); diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index e7ba1eabefab..dab64d5b23d6 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -192,16 +192,25 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t #endif void mod_resource_init(struct processing_module *mod); void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start); +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc); #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall struct vregion *module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size); +__syscall void module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); __syscall int mod_free(struct processing_module *mod, const void *ptr); __syscall void mod_free_all(struct processing_module *mod); #else +struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size); +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); int z_impl_mod_free(struct processing_module *mod, const void *ptr); void z_impl_mod_free_all(struct processing_module *mod); +#define module_adapter_vreg_new z_impl_module_adapter_vreg_new +#define module_adapter_vreg_unmap z_impl_module_adapter_vreg_unmap #define mod_alloc_ext z_impl_mod_alloc_ext #define mod_free z_impl_mod_free #define mod_free_all z_impl_mod_free_all diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index b727d1e700fb..d512ccdff2ef 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -167,6 +167,8 @@ size_t get_shared_buffer_heap_size(void); struct mod_alloc_ctx { struct k_heap *heap; struct vregion *vreg; + uintptr_t vreg_start; + size_t vreg_size; }; /** From 10d87da9d68eb0cc7abc8cdbc8b96067979df9bc Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 10:50:58 +0200 Subject: [PATCH 11/30] vregion: make 3 vregion API functions syscalls vregion_get(), vregion_put() and vregion_set_interim() should also be callable from the userspace. Make them syscalls. Also remove redundant symbol exporting since the vregion API shouldn't be used directly by LLEXT modules. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib/vregion.h | 8 +++--- zephyr/CMakeLists.txt | 1 + zephyr/lib/vregion.c | 51 +++++++++++++++++++++++++++++++---- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 5c066c90dbc8..7d0bd11b27da 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -49,7 +49,7 @@ struct vregion *vregion_create(size_t memsize); * * @param[in] vr Pointer to the virtual region instance. */ -void vregion_set_interim(struct vregion *vr); +__syscall void vregion_set_interim(struct vregion *vr); /** * @brief Increment virtual region's user count. @@ -60,7 +60,7 @@ void vregion_set_interim(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance. */ -struct vregion *vregion_get(struct vregion *vr); +__syscall struct vregion *vregion_get(struct vregion *vr); /** * @brief Decrement virtual region's user count or destroy it. @@ -71,7 +71,7 @@ struct vregion *vregion_get(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr); +__syscall struct vregion *vregion_put(struct vregion *vr); /** * @brief Allocate memory from the specified virtual region. @@ -131,6 +131,8 @@ void vregion_info(struct vregion *vr); */ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); +#include + #else /* CONFIG_SOF_VREGIONS */ struct vregion { diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index f09d94524573..84c7cb35f73f 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -630,6 +630,7 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 9c8c94c23f97..3eaee8e8e3b8 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -161,12 +162,12 @@ struct vregion *vregion_create(size_t memsize) /* log the new vregion */ LOG_INF("new at base %p size %#zx pages %u metadata at %p", - (void *)vr->base, total_size, pages, (void *)vr); + (void *)vregion_base, total_size, pages, (void *)vr); return vr; } -struct vregion *vregion_get(struct vregion *vr) +struct vregion *z_impl_vregion_get(struct vregion *vr) { if (!vr) return NULL; @@ -184,7 +185,7 @@ struct vregion *vregion_get(struct vregion *vr) * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr) +struct vregion *z_impl_vregion_put(struct vregion *vr) { unsigned int use_count; @@ -259,7 +260,7 @@ static void interim_heap_init(struct vregion *vr) vr->lifetime.used = (uint8_t *)vr->lifetime.ptr - (uint8_t *)vr->lifetime.base; } -void vregion_set_interim(struct vregion *vr) +void z_impl_vregion_set_interim(struct vregion *vr) { if (!vr) return; @@ -271,6 +272,47 @@ void vregion_set_interim(struct vregion *vr) k_mutex_unlock(&vr->lock); } +#ifdef CONFIG_USERSPACE +#include +static bool vregion_verify(struct vregion *vr) +{ + if (!vr) + return false; + + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return true; +} + +struct vregion *z_vrfy_vregion_get(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_get(vr); + return NULL; +} +#include + +struct vregion *z_vrfy_vregion_put(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_put(vr); + return NULL; +} +#include + +void z_vrfy_vregion_set_interim(struct vregion *vr) +{ + if (vregion_verify(vr)) + z_impl_vregion_set_interim(vr); +} +#include +#endif + /** * @brief Allocate memory with alignment from the virtual region dynamic heap. * @@ -488,7 +530,6 @@ void vregion_info(struct vregion *vr) LOG_INF("lifetime used %#zx free count %d", vr->lifetime.used, vr->lifetime.free_count); } -EXPORT_SYMBOL(vregion_info); void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) { From 92052fb5cbc5e0241f3610d971d6d3f77988a38d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 20 Aug 2026 14:34:01 +0200 Subject: [PATCH 12/30] schedule: dp: use scheduler_get_user_data() The DP scheduler runs tasks in userspace mode, it's registered with the user scheduler list, therefore it should use scheduler_get_user_data(), not scheduler_get_data(). Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_dp_schedule.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index fa8bbd285791..bfa7c7126cce 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -227,7 +227,7 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) void scheduler_dp_ll_tick(void) { unsigned int lock_key; - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); if (!dp_sch) return; @@ -376,8 +376,7 @@ void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_ unsigned int lock_key; scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_DP; - struct scheduler_dp_data *dp_sch = - (struct scheduler_dp_data *)scheduler_get_data(SOF_SCHEDULE_DP); + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); lock_key = scheduler_dp_lock(cpu_get_id()); scheduler_get_task_info(scheduler_props, data_off_size, &dp_sch->tasks); From de871954437f21919aeda976a8293d48ab308dfd Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 12:45:00 +0200 Subject: [PATCH 13/30] schedule: userspace: DP: make scheduler_dp_ll_tick() a syscall scheduler_dp_ll_tick(() has to recalculate DP deadlines and reschedule DP threads. Make it a syscall to be able to call it from the userspace LL scheduler. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 6 +++++- src/schedule/zephyr_dp_schedule.c | 13 +++++++++++-- src/schedule/zephyr_ll.c | 2 +- zephyr/CMakeLists.txt | 1 + 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 1b55405d0fb1..26774f343acf 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -78,7 +78,11 @@ int scheduler_dp_task_init(struct task **task, uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_ll_tick(void); + +#if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +__syscall void scheduler_dp_ll_tick(unsigned int core); +#include +#endif /** * \brief Extract information about scheduler's tasks diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index bfa7c7126cce..070d3f0ad763 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -224,7 +224,7 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void) +void z_impl_scheduler_dp_ll_tick(unsigned int core) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); @@ -235,11 +235,20 @@ void scheduler_dp_ll_tick(void) /* remember current timestamp as "NOW" */ dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - lock_key = scheduler_dp_lock(cpu_get_id()); + lock_key = scheduler_dp_lock(core); scheduler_dp_recalculate(dp_sch); scheduler_dp_unlock(lock_key); } +#ifdef CONFIG_USERSPACE +#include +void z_vrfy_scheduler_dp_ll_tick(unsigned int core) +{ + z_impl_scheduler_dp_ll_tick(core); +} +#include +#endif + #if CONFIG_SOF_USERSPACE_APPLICATION static int scheduler_dp_task_cancel(void *data, struct task *task) { diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 61e852fb9c92..427193ac2d33 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -326,7 +326,7 @@ static void zephyr_ll_run(void *data) zephyr_ll_unlock(sch, &flags); #ifdef CONFIG_ZEPHYR_DP_SCHEDULER - scheduler_dp_ll_tick(); + scheduler_dp_ll_tick(sch->core); #endif } diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 84c7cb35f73f..4f0813d582ed 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -632,6 +632,7 @@ zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc. zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/dp_schedule.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) From f9ea0dbf4f4a3fe40715e859b852857a7603d0cf Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 17:03:02 +0200 Subject: [PATCH 14/30] schedule: dp: userspace: make scheduler_dp_internal_free() a syscall Make scheduler_dp_internal_free() a syscall in the "application" DP implementation. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 6 +++ src/schedule/zephyr_dp_schedule.h | 4 -- src/schedule/zephyr_dp_schedule_application.c | 41 ++++++++++++++++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 26774f343acf..ce957621ca8a 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -80,6 +80,12 @@ int scheduler_dp_task_init(struct task **task, uint32_t options); #if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +#if CONFIG_SOF_USERSPACE_APPLICATION +__syscall void scheduler_dp_internal_free(struct task *task); +#else +void scheduler_dp_internal_free(struct task *task); +#endif + __syscall void scheduler_dp_ll_tick(unsigned int core); #include #endif diff --git a/src/schedule/zephyr_dp_schedule.h b/src/schedule/zephyr_dp_schedule.h index 694bb541f87e..2119e44c9dfc 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/schedule/zephyr_dp_schedule.h @@ -57,7 +57,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3); unsigned int scheduler_dp_lock(uint16_t core); void scheduler_dp_unlock(unsigned int key); void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_internal_free(struct task *task); diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index daf9070ae4dc..2e600b449700 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -400,7 +401,7 @@ struct scheduler_dp_task_memory { struct ipc4_flat flat; }; -void scheduler_dp_internal_free(struct task *task) +void z_impl_scheduler_dp_internal_free(struct task *task) { struct task_dp_pdata *pdata = task->priv_data; @@ -615,3 +616,41 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, mod_free(mod, task_memory); return ret; } + +#ifdef CONFIG_USERSPACE +#include + +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + assert(alloc); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc->vreg_start, alloc->vreg_size)); +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + + struct task_dp_pdata *pdata = task->priv_data; + + K_OOPS(K_SYSCALL_OBJ(pdata->event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(pdata->thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(pdata->mod); + return z_impl_scheduler_dp_internal_free(task); +} +#include +#endif From 4315e8d3424a2242f61d3ebf381070048c087a36 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 16:18:53 +0200 Subject: [PATCH 15/30] schedule: ll: enable multicore userspace Make scheduling LL thread and synchronisation objects per-core and forward IPCs and scheduling events accordingly. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/ipc/common.h | 17 +++-- src/init/init.c | 10 ++- src/ipc/ipc-common.c | 123 ++++++++++++++++++++++++++--------- src/ipc/ipc4/handler-user.c | 103 +++++++++++++++++++++++------ uuid-registry.txt | 1 + 5 files changed, 199 insertions(+), 55 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index a910c6d42c92..55c668d199f3 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -57,9 +57,9 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_POWERDOWN BIT(3) struct ipc_user { - struct k_thread *thread; + struct k_thread *thread[CONFIG_CORE_COUNT]; struct k_sem *sem; - struct k_event *event; + struct k_event *event[CONFIG_CORE_COUNT]; /** @brief Copy of IPC4 message primary word forwarded to user thread */ uint32_t ipc_msg_pri; /** @brief Copy of IPC4 message extension word forwarded to user thread */ @@ -73,9 +73,10 @@ struct ipc_user { /** @brief Reply TX data pointer from user thread (e.g. LARGE_CONFIG_GET result) */ void *reply_tx_data; struct ipc *ipc; - struct k_thread *audio_thread; + struct k_thread *audio_thread[CONFIG_CORE_COUNT]; /** @brief Original kernel driver pointer for restoring dev->drv after create */ const struct comp_driver *init_drv; + bool init_needed[CONFIG_CORE_COUNT]; /** * @brief User-accessible copy of comp_driver + tr_ctx for create(). * @@ -326,7 +327,7 @@ extern bool ipc_enter_gdb; * @param extension Extension message word * @return Result code from user thread processing */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension); +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core); /** * @brief Protocol-specific dispatch of a forwarded IPC command. @@ -338,6 +339,14 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension); * @return Result code to report back to the host */ int ipc_user_thread_dispatch(struct ipc_user *ipc_user); + +/** + * @brief Initialize IPC and LL scheduler threads on a booting secondary core. + * + * @param core Secondary core ID + * @return 0 or a negative error code + */ +int ipc_user_init_secondary(unsigned int core); #endif #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/init/init.c b/src/init/init.c index 5990cfebc2dc..ba650dfc5256 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,7 @@ static inline int secondary_core_restore(void) { return 0; }; __cold int secondary_core_init(struct sof *sof) { + unsigned int core = cpu_get_id(); int err; struct ll_schedule_domain *dma_domain; @@ -134,6 +136,12 @@ __cold int secondary_core_init(struct sof *sof) if (dma_domain) scheduler_init_ll(dma_domain); +#if CONFIG_SOF_USERSPACE_LL + err = ipc_user_init_secondary(core); + if (err < 0) + return err; +#endif + #if CONFIG_ZEPHYR_DP_SCHEDULER err = scheduler_dp_init(); if (err < 0) @@ -152,7 +160,7 @@ __cold int secondary_core_init(struct sof *sof) return err; #endif #if CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL - err = core_kcps_adjust(cpu_get_id(), SECONDARY_CORE_BASE_CPS_USAGE); + err = core_kcps_adjust(core, SECONDARY_CORE_BASE_CPS_USAGE); if (err < 0) return err; #endif diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 0e7b344510cd..a49ceffd5b1a 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -318,7 +318,10 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) -static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); /** * @brief Forward an IPC command to the user-space thread. @@ -332,7 +335,7 @@ static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SI * @param extension Extension message word * @return Result from user thread processing */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension) +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -346,13 +349,22 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension) pdata->ipc_msg_ext = extension; pdata->ipc = ipc; + /* + * Forwarding the first IPC to this core, wait for its userspace IPC + * thread to start + */ + if (pdata->init_needed[core]) { + pdata->init_needed[core] = false; + k_sem_take(pdata->sem, K_FOREVER); + } + /* Prevent host completion until user thread finishes */ key = k_spin_lock(&ipc->lock); ipc->task_mask |= IPC_TASK_IN_THREAD; k_spin_unlock(&ipc->lock, key); /* Wake the user thread */ - k_event_set(pdata->event, IPC_USER_EVENT_CMD); + k_event_set(pdata->event[core], IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ ret = k_sem_take(pdata->sem, K_MSEC(100)); @@ -389,8 +401,8 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) static void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; + unsigned int core = POINTER_TO_UINT(p2); - ARG_UNUSED(p2); ARG_UNUSED(p3); __ASSERT(k_is_user_context(), "expected user context"); @@ -400,7 +412,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) LOG_INF("IPC user-space thread started"); for (;;) { - uint32_t mask = k_event_wait_safe(ipc_user->event, + uint32_t mask = k_event_wait_safe(ipc_user->event[core], IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, false, K_FOREVER); @@ -418,7 +430,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) } } -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user) +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) { char thread_name[] = "ll_user0"; int ret; @@ -426,47 +438,85 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) assert_can_be_cold(); /* Allocate kernel objects for the user-space thread */ - ipc_user->event = k_object_alloc(K_OBJ_EVENT); - if (!ipc_user->event) { + ipc_user->event[core] = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event[core]) { LOG_ERR("user IPC event alloc failed"); return -ENOMEM; } - k_event_init(ipc_user->event); + k_event_init(ipc_user->event[core]); - ipc_user->thread = k_object_alloc(K_OBJ_THREAD); - if (!ipc_user->thread) { + ipc_user->thread[core] = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->thread[core]) { LOG_ERR("user IPC thread alloc failed"); ret = -ENOMEM; goto e_event; } - k_thread_create(ipc_user->thread, ipc_user_stack, + k_thread_create(ipc_user->thread[core], ipc_user_stack[core], CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, NULL, NULL, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, -1, K_USER, K_FOREVER); - k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - k_thread_name_set(ipc_user->thread, thread_name); + k_thread_cpu_pin(ipc_user->thread[core], core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(ipc_user->thread[core], thread_name); /* * Each userspace IPC thread must be able to wait on its private event * and signal completion on the primary core semaphore */ - k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event); - user_grant_dai_access_all(ipc_user->thread); - user_grant_dma_access_all(ipc_user->thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); - user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - pipeline_posn_grant_access(ipc_user->thread); + k_thread_access_grant(ipc_user->thread[core], ipc_user->sem, ipc_user->event[core]); + user_grant_dai_access_all(ipc_user->thread[core]); + user_grant_dma_access_all(ipc_user->thread[core]); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread[core]); + user_ll_grant_access(ipc_user->thread[core], core); + pipeline_posn_grant_access(ipc_user->thread[core]); return 0; e_event: - k_object_free(ipc_user->event); + k_object_free(ipc_user->event[core]); return ret; } +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + int ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user->thread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + ipc_user->audio_thread[core] = scheduler_init_context(task); + if (!ipc_user->audio_thread[core]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user->thread[core], ipc_user->audio_thread[core]); + ipc_user->init_needed[core] = true; + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + return 0; +} + +/* Primary core only */ __cold static void ipc_user_init(void) { struct ipc *ipc = ipc_get(); @@ -480,6 +530,8 @@ __cold static void ipc_user_init(void) sof_panic(SOF_IPC_PANIC_IPC); } + assert_can_be_cold(); + ipc_user->sem = k_object_alloc(K_OBJ_SEM); if (!ipc_user->sem) { LOG_ERR("user IPC sem alloc failed"); @@ -530,13 +582,14 @@ __cold static void ipc_user_init(void) k_sem_init(ipc_user->sem, 0, 1); - ret = ipc_user_init_thread(ipc_user); + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); if (ret < 0) { LOG_ERR("user IPC thread initialization failed"); sof_panic(SOF_IPC_PANIC_IPC); } - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread); + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); if (ret < 0) { LOG_ERR("ipc user: mailbox access grant failed: %d", ret); sof_panic(SOF_IPC_PANIC_IPC); @@ -545,22 +598,32 @@ __cold static void ipc_user_init(void) /* Store references in ipc struct so kernel handler can forward commands */ ipc->ipc_user_pdata = ipc_user; - k_thread_start(ipc_user->thread); - struct task *task = zephyr_ll_task_alloc(); + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, cpu_get_id(), 0); - ipc_user->audio_thread = scheduler_init_context(task); + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = scheduler_init_context(task); + if (!ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } /* Grant ipc_user thread permission on the audio thread object. * Needed so user-space dai_common_new() can call * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ - k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread); + k_thread_access_grant(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID], + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]); + + k_thread_start(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); + k_sem_take(ipc_user->sem, K_FOREVER); } #else static void ipc_user_init(void) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 13954d7929e9..66c2b4e85af7 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -90,7 +90,31 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifndef CONFIG_SOF_USERSPACE_LL +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} +#else __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) { struct ipc *ipc = ipc_get(); @@ -99,9 +123,7 @@ __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); } -#endif -#ifndef CONFIG_SOF_USERSPACE_LL __cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) { struct ipc4_pipeline_delete *pipe; @@ -686,6 +708,7 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + struct ipc *ipc = ipc_get(); uint32_t type; int ret; @@ -709,21 +732,57 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, /* pipeline settings */ case SOF_IPC4_GLB_CREATE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); + } #else ret = ipc4_new_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_DELETE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } #else ret = ipc4_delete_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_SET_PIPELINE_STATE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + int id = ipc4_pipeline_id_get(ipc4, &state, NULL, NULL); + if (id < 0) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } #else ret = ipc4_set_pipeline_state(ipc4); #endif @@ -1525,14 +1584,11 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, * access to IMR manifest and driver list in kernel memory). * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. - * Cross-core creation stays fully in kernel. */ struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv( - IPC4_MOD_ID(comp_id)); - struct ipc_user *pdata = ipc->ipc_user_pdata; + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); if (!drv) { ret = IPC4_MOD_NOT_INITIALIZED; @@ -1542,6 +1598,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, /* Copy comp_driver and tr_ctx into user-accessible ipc_user buffer * originals are in kernel .rodata/.data and not readable from user mode. */ + struct ipc_user *pdata = ipc->ipc_user_pdata; struct comp_driver *drv_copy = (struct comp_driver *)pdata->init_drv_data; struct tr_ctx *tctx_copy = (struct tr_ctx *)(pdata->init_drv_data + @@ -1558,7 +1615,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; pdata->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); #endif } else { ret = ipc4_init_module_instance(ipc4); @@ -1567,7 +1625,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1581,7 +1640,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_SET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_set_get_config_module_instance(ipc4, true); #endif @@ -1594,8 +1654,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, if (config->primary.r.module_id) { /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, - ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1623,8 +1683,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, (const struct ipc4_module_large_config *)ipc4; if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, - ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); } else { /* Base firmware: keep in kernel (IMR access) */ ret = ipc4_set_large_config_module_instance(ipc4); @@ -1636,21 +1696,24 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; case SOF_IPC4_MOD_BIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_bind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_UNBIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_unbind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_DELETE_INSTANCE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_delete_module_instance(ipc4); #endif diff --git a/uuid-registry.txt b/uuid-registry.txt index e9e8f8e77876..b4d3f437e9b1 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -147,6 +147,7 @@ d7f6712d-131c-45a7-82ed6aa9dc2291ea pm_runtime 9302adf5-88be-4234-a0a7dca538ef81f4 sai 3dee06de-f25a-4e10-ae1fabc9573873ea schedule 70d223ef-2b91-4aac-b444d89a0db2793a sdma +bdcb1461-34f5-4047-b9cc70fdf8dfb234 sec_core_init 55a88ed5-3d18-46ca-88f10ee6eae9930f selector 32fe92c1-1e17-4fc2-9758c7f3542e980a selector4 cf90d851-68a2-4987-a2de85aed0c8531c sgen_mt8186 From f0af694ed3d6f758e9ca3874ebdb01cc7213c3f2 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 15:22:17 +0200 Subject: [PATCH 16/30] schedule: dp: userspace grant IPC thread rights on DP assets In case of userspace LL scheduling the (also userspace) IPC thread needs access rights to DP assets like the thread itself and its stack and synchronisation primitives. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/ipc/common.h | 5 +++++ src/ipc/ipc-common.c | 8 ++++++++ src/schedule/zephyr_dp_schedule_application.c | 14 +++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 55c668d199f3..328a5e85327f 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -349,4 +349,9 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user); int ipc_user_init_secondary(unsigned int core); #endif +/** + * \brief get pointer to the userspace IPC thread for core + */ +struct k_thread *ipc_thread_user(unsigned int core); + #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index a49ceffd5b1a..dd1a1ee6ee01 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -516,6 +516,14 @@ __cold int ipc_user_init_secondary(unsigned int core) return 0; } +struct k_thread *ipc_thread_user(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + + return ipc_user->thread[core]; +} + /* Primary core only */ __cold static void ipc_user_init(void) { diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 2e600b449700..cd4a61c37f5e 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -156,6 +157,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, const union scheduler_dp_thread_ipc_param *param) { struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + unsigned int core = pmod->dev->task->core; int ret; if (!pmod) { @@ -165,14 +167,14 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { /* Wait for the DP thread to start */ - ret = k_sem_take(&dp_sync[pmod->dev->task->core], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) { tr_err(&dp_tr, "Failed waiting for DP thread to start: %d", ret); return ret; } } - unsigned int lock_key = scheduler_dp_lock(pmod->dev->task->core); + unsigned int lock_key = scheduler_dp_lock(core); /* IPCs are serialised */ pdata->flat->ret = -ENOSYS; @@ -185,7 +187,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (!ret) { /* Wait for completion */ - ret = k_sem_take(&dp_sync[cpu_get_id()], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) tr_err(&dp_tr, "Failed waiting for DP thread: %d", ret); else @@ -527,6 +529,12 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(pdata->thread_id, pdata->event, &dp_sync[core]); scheduler_dp_grant(pdata->thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, pdata->event, pdata->thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); +#endif struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), SOF_MEM_FLAG_COHERENT); From 93518f0920588415535b6c4a6a25a977c81ddfca Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 16:17:54 +0200 Subject: [PATCH 17/30] schedule: ll: userspace: grant the LL thread rights on DP The LL userspace thread has to interact with the DP one. Grant required rights. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/ll_schedule_domain.h | 1 + src/schedule/zephyr_domain.c | 12 ++++++++++++ src/schedule/zephyr_dp_schedule_application.c | 1 + 3 files changed, 14 insertions(+) diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 03991debb0cb..cfd0043995db 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -330,6 +330,7 @@ struct ll_schedule_domain *zephyr_domain_init(int clk); struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain); struct k_thread *zephyr_domain_thread_tid_for_core(int core); struct k_mem_domain *zephyr_ll_mem_domain(void); +struct k_thread *zephyr_ll_domain_thread(void); #endif /* CONFIG_SOF_USERSPACE_LL */ #ifdef CONFIG_SOF_FULL_ZEPHYR_APPLICATION __syscall int zephyr_ll_task_sem_alloc(struct task *task); diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index b37f91285854..cc1b76600dee 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -525,6 +525,18 @@ struct k_thread *zephyr_domain_thread_tid_for_core(int core) return ll_thread_tid[core]; } +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index cd4a61c37f5e..1f9760421912 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -534,6 +534,7 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(thread_ipc, pdata->event, pdata->thread_id, p_stack, &dp_sync[core]); scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(), core); #endif struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), From c7110ae74645620a755ad92c75618199cece5dbe Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 9 Jul 2026 12:16:24 +0200 Subject: [PATCH 18/30] ipc: add a comment to explain DP flow Switching to the userspace mode in DP and LL cases differs. Add a comment to explain that. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/handler-user.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 66c2b4e85af7..bd2853a8141c 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1619,6 +1619,11 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, mi->extension.r.core_id); #endif } else { + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ ret = ipc4_init_module_instance(ipc4); } break; From ebdf98c87290b50a2310ff502dadf7d9b7bb66fa Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 09:26:48 +0200 Subject: [PATCH 19/30] ipc: (cosmetic) compact several lines Join several needlessly broken lines. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc-helper.c | 21 +++++++-------------- src/ipc/ipc4/helper.c | 43 ++++++++++++++----------------------------- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 567255a9e22a..4c46d104856f 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -82,8 +82,7 @@ __cold struct comp_buffer *buffer_new(struct mod_alloc_ctx *alloc, #endif /* allocate buffer */ - buffer = buffer_alloc(alloc, desc->size, flags, PLATFORM_DCACHE_ALIGN, - is_shared); + buffer = buffer_alloc(alloc, desc->size, flags, PLATFORM_DCACHE_ALIGN, is_shared); if (buffer) { buffer->stream.runtime_stream_params.id = desc->comp.id; buffer->stream.runtime_stream_params.pipeline_id = desc->comp.pipeline_id; @@ -188,14 +187,12 @@ int comp_verify_params(struct comp_dev *dev, uint32_t flag, if (dir == PPL_DIR_DOWNSTREAM) { comp_dev_for_each_consumer(dev, buf) { comp_update_params(flag, params, buf); - buffer_set_params(buf, params, - BUFFER_UPDATE_FORCE); + buffer_set_params(buf, params, BUFFER_UPDATE_FORCE); } } else { comp_dev_for_each_producer(dev, buf) { comp_update_params(flag, params, buf); - buffer_set_params(buf, params, - BUFFER_UPDATE_FORCE); + buffer_set_params(buf, params, BUFFER_UPDATE_FORCE); } } @@ -284,11 +281,9 @@ int ipc_pipeline_complete(struct ipc *ipc, uint32_t comp_id) pipeline_id = ipc_pipe->pipeline->pipeline_id; - tr_dbg(&ipc_tr, "ipc: pipe %d -> complete on comp 0x%x", pipeline_id, - comp_id); + tr_dbg(&ipc_tr, "ipc: pipe %d -> complete on comp 0x%x", pipeline_id, comp_id); - return pipeline_complete(ipc_pipe->pipeline, ipc_ppl_source->cd, - ipc_ppl_sink->cd); + return pipeline_complete(ipc_pipe->pipeline, ipc_ppl_source->cd, ipc_ppl_sink->cd); } __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) @@ -306,8 +301,7 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) /* check whether component exists */ icd = ipc_get_comp_by_id(ipc, comp_id); if (!icd) { - tr_err(&ipc_tr, "comp id: 0x%x is not found", - comp_id); + tr_err(&ipc_tr, "comp id: 0x%x is not found", comp_id); return -ENODEV; } @@ -336,8 +330,7 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) * leak on error. Bug-free host drivers won't do * this, this was found via fuzzing. */ - tr_err(&ipc_tr, "uninitialized buffer lists on comp 0x%x\n", - icd->id); + tr_err(&ipc_tr, "uninitialized buffer lists on comp 0x%x\n", icd->id); return -EINVAL; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 476fbf438d8b..8edbdb350a6f 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -182,9 +182,7 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; } else { - tr_err(&ipc_tr, - "ipc: DP scheduling is disabled, cannot create comp 0x%x", - comp_id); + tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); return NULL; } @@ -1160,8 +1158,7 @@ __cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); dev->ipc_config.id = comp_id; - dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id - + IPC4_MAX_MODULE_COUNT; + dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; return ipc4_add_comp_dev(dev); } @@ -1303,13 +1300,10 @@ __cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) /* search driver list with UUID */ list_for_item(clist, &drivers->list) { - info = container_of(clist, struct comp_driver_info, - list); + info = container_of(clist, struct comp_driver_info, list); if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { - tr_dbg(&comp_tr, - "found type %d, uuid %pU", - info->drv->type, - info->drv->tctx->uuid_p); + tr_dbg(&comp_tr, "found type %d, uuid %pU", + info->drv->type, info->drv->tctx->uuid_p); drv = info->drv; break; } @@ -1391,8 +1385,7 @@ static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) return inf->drv; } - tr_err(&comp_tr, - "no instantiable driver at module_id %u (%u available)", + tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", module_id, idx); return NULL; } @@ -1636,10 +1629,8 @@ void ipc4_audio_format_to_stream_params(const struct ipc4_audio_format *audio_fm params->sample_valid_bytes = audio_fmt->valid_bit_depth / 8; params->buffer_fmt = audio_fmt->interleaving_style; - audio_stream_fmt_conversion(audio_fmt->depth, - audio_fmt->valid_bit_depth, - &frame_fmt, &valid_fmt, - audio_fmt->s_type); + audio_stream_fmt_conversion(audio_fmt->depth, audio_fmt->valid_bit_depth, + &frame_fmt, &valid_fmt, audio_fmt->s_type); params->frame_fmt = frame_fmt; for (i = 0; i < SOF_IPC_MAX_CHANNELS; i++) @@ -1664,10 +1655,8 @@ void ipc4_update_buffer_format(struct comp_buffer *buf_c, audio_stream_set_channels(&buf_c->stream, fmt->channels_count); audio_stream_set_rate(&buf_c->stream, fmt->sampling_frequency); - audio_stream_fmt_conversion(fmt->depth, - fmt->valid_bit_depth, - &frame_fmt, &valid_fmt, - fmt->s_type); + audio_stream_fmt_conversion(fmt->depth, fmt->valid_bit_depth, + &frame_fmt, &valid_fmt, fmt->s_type); audio_stream_set_frm_fmt(&buf_c->stream, frame_fmt); audio_stream_set_valid_fmt(&buf_c->stream, valid_fmt); @@ -1686,10 +1675,8 @@ void ipc4_update_source_format(struct sof_source *source, source_set_channels(source, fmt->channels_count); source_set_rate(source, fmt->sampling_frequency); - audio_stream_fmt_conversion(fmt->depth, - fmt->valid_bit_depth, - &frame_fmt, &valid_fmt, - fmt->s_type); + audio_stream_fmt_conversion(fmt->depth, fmt->valid_bit_depth, + &frame_fmt, &valid_fmt, fmt->s_type); source_set_frm_fmt(source, frame_fmt); source_set_valid_fmt(source, valid_fmt); @@ -1704,10 +1691,8 @@ void ipc4_update_sink_format(struct sof_sink *sink, sink_set_channels(sink, fmt->channels_count); sink_set_rate(sink, fmt->sampling_frequency); - audio_stream_fmt_conversion(fmt->depth, - fmt->valid_bit_depth, - &frame_fmt, &valid_fmt, - fmt->s_type); + audio_stream_fmt_conversion(fmt->depth, fmt->valid_bit_depth, + &frame_fmt, &valid_fmt, fmt->s_type); sink_set_frm_fmt(sink, frame_fmt); sink_set_valid_fmt(sink, valid_fmt); From 47fa5c461fefab31be771d65c518d6343d0cd513 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 10:48:07 +0200 Subject: [PATCH 20/30] ipc: ipc4: extract common code into a function Extract common initialisation code from comp_new_ipc4_user() and comp_new_ipc4() into a new function. Signed-off-by: Guennadi Liakhovetski --- src/ipc/ipc4/helper.c | 118 ++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 69 deletions(-) diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 8edbdb350a6f..27039021c093 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -111,42 +111,65 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - uint32_t comp_id; - unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); assert_can_be_cold(); - comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - if (ipc4_get_comp_dev(comp_id)) { tr_err(&ipc_tr, "comp 0x%x exists", comp_id); - return NULL; + return -EEXIST; } if (module_init->extension.r.core_id >= CONFIG_CORE_COUNT) { tr_err(&ipc_tr, "ipc: comp->core = %u", (uint32_t)module_init->extension.r.core_id); - return NULL; + return -EINVAL; } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init->extension.r.ppl_instance_id; - ipc_config.core = module_init->extension.r.core_id; - ipc_config.ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init->extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { + memset(ipc_config, 0, sizeof(*ipc_config)); + ipc_config->id = comp_id; + ipc_config->pipeline_id = module_init->extension.r.ppl_instance_id; + ipc_config->core = module_init->extension.r.core_id; + ipc_config->ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); + ipc_config->ipc_extended_init = module_init->extension.r.extended_init; + if (ipc_config->ipc_config_size > MAILBOX_HOSTBOX_SIZE) { tr_err(&ipc_tr, "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; + ipc_config->ipc_config_size); + return -ENOSPC; + } + + if (!module_init->extension.r.proc_domain) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_LL; + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { + tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); + return -EINVAL; } + return 0; +} + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + /* Reject a module naming a non-existent parent pipeline: otherwise * dev->pipeline stays NULL and a later init path (e.g. the copier) * dereferences it. IPC4_INVALID_PIPELINE_ID is exempt - it marks base FW @@ -160,6 +183,7 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo (uint32_t)ipc_config.pipeline_id); return NULL; } + #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range((__sparse_force void __sparse_cache *) @@ -177,15 +201,6 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (!drv) return NULL; - if (!module_init->extension.r.proc_domain) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; - } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - } else { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); - return NULL; - } - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, @@ -238,43 +253,19 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, struct ipc4_module_init_instance module_init; struct comp_ipc_config ipc_config; struct comp_dev *dev; - uint32_t comp_id; unsigned char *data; - int ret; assert_can_be_cold(); - ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ret < 0) - return NULL; - - comp_id = IPC4_COMP_ID(module_init.primary.r.module_id, - module_init.primary.r.instance_id); + int ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ipc4_get_comp_dev(comp_id)) { - tr_err(&ipc_tr, "comp 0x%x exists", comp_id); + if (ret < 0) return NULL; - } - if (module_init.extension.r.core_id >= CONFIG_CORE_COUNT) { - tr_err(&ipc_tr, "ipc: comp->core = %u", - (uint32_t)module_init.extension.r.core_id); + ret = ipc4_comp_new_config(&ipc_config, &module_init); + if (ret < 0) return NULL; - } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init.extension.r.ppl_instance_id; - ipc_config.core = module_init.extension.r.core_id; - ipc_config.ipc_config_size = - module_init.extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init.extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { - tr_err(&ipc_tr, - "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; - } #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range( @@ -284,17 +275,6 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, #endif data = ipc4_get_comp_new_data(); - if (!module_init.extension.r.proc_domain) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; - } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - } else { - tr_err(&ipc_tr, - "ipc: DP scheduling is disabled, cannot create comp 0x%x", - comp_id); - return NULL; - } - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, From d7462d1f6ab7520bc2790585d47563919d5e8cb6 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 14:01:11 +0200 Subject: [PATCH 21/30] llext: move 2 functions higher Move llext_manager_add_partition() and llext_manager_rm_partition() higher in the file for future use. No functional change. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 61 +++++++++++++++-------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index d1f6740d1eaf..64ae90e30ed2 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -168,6 +168,38 @@ static int llext_manager_load_data_from_storage(const struct sys_mm_drv_region * return ret; } +#ifdef CONFIG_USERSPACE +static int llext_manager_add_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "add %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_add_partition(domain, &part); +} + +static int llext_manager_rm_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "remove %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_remove_partition(domain, &part); +} +#endif + static void llext_manager_unmap_detached_sections(const struct llext_loader *ldr, const struct llext *ext, enum llext_mem region, @@ -791,35 +823,6 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config } #ifdef CONFIG_USERSPACE -static int llext_manager_add_partition(struct k_mem_domain *domain, - uintptr_t addr, size_t size, - k_mem_partition_attr_t attr) -{ - size_t pre_pad_size = addr & (PAGE_SZ - 1); - struct k_mem_partition part = { - .start = addr - pre_pad_size, - .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), - .attr = attr, - }; - - tr_dbg(&lib_manager_tr, "add %#zx @ %lx partition", part.size, part.start); - return k_mem_domain_add_partition(domain, &part); -} - -static int llext_manager_rm_partition(struct k_mem_domain *domain, - uintptr_t addr, size_t size, - k_mem_partition_attr_t attr) -{ - size_t pre_pad_size = addr & (PAGE_SZ - 1); - struct k_mem_partition part = { - .start = addr - pre_pad_size, - .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), - .attr = attr, - }; - - tr_dbg(&lib_manager_tr, "remove %#zx @ %lx partition", part.size, part.start); - return k_mem_domain_remove_partition(domain, &part); -} static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) { From 3780cd732f245e99a2f645a94947c5eb02277fca Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 14:48:45 +0200 Subject: [PATCH 22/30] llext: don't access DRAM when freeing With userspace LL enabled module freeing runs in syscall contex on behalf of the userspace IPC thread. That thread doesn't have access to DRAM. Therefore we cannot call lib_manager_get_module_manifest() in that case. Use SRAM module data by calling llext_manager_mod_find() instead. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/llext_manager.h | 3 +++ src/library_manager/lib_manager.c | 9 ++++++--- src/library_manager/llext_manager.c | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 525fa50b1506..1d5dc96c2562 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -16,6 +16,7 @@ struct comp_dev; struct comp_driver; struct comp_ipc_config; struct k_mem_domain; +struct lib_manager_mod_ctx; static inline bool module_is_llext(const struct sof_man_module *mod) { @@ -33,6 +34,7 @@ int llext_manager_add_library(uint32_t module_id); int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain); int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain); +int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx); bool comp_is_llext(struct comp_dev *comp); #else #define module_is_llext(mod) false @@ -40,6 +42,7 @@ bool comp_is_llext(struct comp_dev *comp); #define llext_manager_free_module(component_id) 0 #define llext_manager_add_library(module_id) 0 #define llext_manager_add_domain(component_id, domain) 0 +#define llext_manager_mod_find(ctx, idx) -ENOENT #define comp_is_llext(comp) false #endif diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 35f442f47fec..bba4b97841d6 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -423,15 +423,18 @@ int z_impl_lib_manager_free_module(const uint32_t component_id) tr_dbg(&lib_manager_tr, "mod_id: %#x", component_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const struct lib_manager_mod_ctx *const ctx = lib_manager_get_mod_ctx(module_id); + + if (llext_manager_mod_find(ctx, entry_index) >= 0) + return llext_manager_free_module(component_id); + mod = lib_manager_get_module_manifest(module_id); if (!mod) { tr_err(&lib_manager_tr, "failed to get module descriptor"); return -EINVAL; } - if (module_is_llext(mod)) - return llext_manager_free_module(component_id); - ret = lib_manager_unload_module(mod); if (ret < 0) return ret; diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 64ae90e30ed2..187925a5b46c 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -569,7 +569,7 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, } /* Find a module context, containing the driver with the supplied index */ -static int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) +int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) { unsigned int i; From 1a57fb1db4056950e786df35343c9502bf34f0b6 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 15:09:09 +0200 Subject: [PATCH 23/30] llext: temporarily map ELF headers for userspace when freeing llext_manager_unload_module() is called in a syscall context on behalf of the userspace IPC thread, so it doesn't have direct access to LLEXT module DRAM data. Map section headers temporarily for the duration of the function. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 187925a5b46c..ede6c68c0ade 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -380,6 +380,18 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->segment[LIB_MANAGER_BSS].size; int err = 0, ret; + unsigned int sect_cnt = llext_section_count(ext); + size_t total = sect_cnt * sizeof(elf_shdr_t); + const elf_shdr_t *shdr; + + ret = llext_get_section_info(ldr, ext, 0, &shdr, NULL, NULL); + if (ret < 0) + return ret; + + /* Temporarily map ELF section headers */ + llext_manager_add_partition(zephyr_ll_mem_domain(), (uintptr_t)shdr, total, + K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB); + llext_manager_unmap_detached_sections(ldr, ext, LLEXT_MEM_TEXT, va_base_text, text_size); ret = llext_manager_align_unmap(va_base_text, text_size); @@ -404,6 +416,9 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) if (ret < 0 && !err) err = ret; + llext_manager_rm_partition(zephyr_ll_mem_domain(), (uintptr_t)shdr, total, + K_MEM_PARTITION_P_RW_U_NA | XTENSA_MMU_CACHED_WB); + mctx->mapped = false; #ifdef CONFIG_SOF_USERSPACE_LL From d49bf8e833c62ce2fa841ca15aa58b8b27af2be6 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 15:14:59 +0200 Subject: [PATCH 24/30] llext: avoid DRAM access in userspace when freeing llext_manager_free_module() cannot access DRAM when running with userspace LL enabled. Don't call lib_manager_get_library_manifest() to obtain the DRAM descriptor, needed to verify the entry index. The index is now verified by llext_manager_mod_find(). Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index ede6c68c0ade..3dad4626c4f1 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -1081,16 +1081,9 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do int llext_manager_free_module(const uint32_t component_id) { const uint32_t module_id = IPC4_MOD_ID(component_id); - struct sof_man_fw_desc *desc = (struct sof_man_fw_desc *)lib_manager_get_library_manifest(module_id); struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - if (entry_index >= desc->header.num_module_entries) { - tr_err(&lib_manager_tr, "Invalid driver index %u exceeds %d", - entry_index, desc->header.num_module_entries - 1); - return -ENOENT; - } - if (!ctx->mod) { tr_err(&lib_manager_tr, "NULL module array: ID %#x ctx %p", component_id, ctx); return -ENOENT; From 4a66dbdd207c448a3d0d5f2e80d9c4e7115c75c7 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:04:10 +0200 Subject: [PATCH 25/30] userspace: perform library loading in kernel context When running in syscall context on behalf of a userspace thread dynamically mapped memory doesn't automatically become accessible. To make it accessible it has to be added to the thread memory domain. This is a problem for loadable modules with executable cold sections. To be able to execute them they have to be mapped to threads with the executable bit set. While for linking that memory has to be mapped writable. To solve the problem we perform linking from the kernel IPC context before forwarding to the userspace IPC thread. Signed-off-by: Guennadi Liakhovetski --- src/include/ipc4/handler.h | 8 +++++++ src/ipc/ipc4/handler-user.c | 47 ++++++++++++------------------------- src/ipc/ipc4/helper.c | 33 ++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index d6f839458d54..cb54ecc08db0 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -16,6 +16,14 @@ struct ipc4_message_request; */ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * \brief Load a dynamically loadable module. + * @param[in] drv Component driver. + * @param[in] mi SOF_IPC4_MOD_INIT_INSTANCE data + */ +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi); + /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @param[in] ipc4 IPC4 message request. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index bd2853a8141c..fb84a60c4164 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1585,7 +1585,6 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. */ - struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); @@ -1595,26 +1594,18 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; } - /* Copy comp_driver and tr_ctx into user-accessible ipc_user buffer - * originals are in kernel .rodata/.data and not readable from user mode. - */ - struct ipc_user *pdata = ipc->ipc_user_pdata; - struct comp_driver *drv_copy = (struct comp_driver *)pdata->init_drv_data; - struct tr_ctx *tctx_copy = - (struct tr_ctx *)(pdata->init_drv_data + - sizeof(struct comp_driver)); - - ret = memcpy_s(drv_copy, sizeof(*drv_copy), drv, sizeof(*drv)); - if (!ret && drv->tctx) { - ret = memcpy_s(tctx_copy, sizeof(*tctx_copy), - drv->tctx, sizeof(*drv->tctx)); - drv_copy->tctx = tctx_copy; - } + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - if (ret < 0) - break; + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); - pdata->init_drv = drv; + if (err < 0) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + } + + ipc_get()->ipc_user_pdata->init_drv = drv; ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, mi->extension.r.core_id); #endif @@ -1788,36 +1779,28 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * module code does not execute with kernel privileges. * * init_drv = original kernel pointer - * init_drv_data = user-accessible copy */ - const struct comp_driver *orig_drv = ipc_user->init_drv; - const struct comp_driver *drv_copy = - (const struct comp_driver *)ipc_user->init_drv_data; - struct comp_dev *dev; + const struct comp_driver *drv = ipc_user->init_drv; ipc_user->init_drv = NULL; - if (!orig_drv) { + if (!drv) { result = IPC4_MOD_NOT_INITIALIZED; break; } - dev = comp_new_ipc4_user(&msg, drv_copy); + struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); + if (!dev) { result = IPC4_MOD_NOT_INITIALIZED; break; } - /* Restore original kernel driver pointer. comp_init() - * set dev->drv to the copy; runtime code expects the - * canonical kernel address. - */ - dev->drv = orig_drv; - result = ipc4_add_comp_dev(dev); if (result != IPC4_SUCCESS) break; comp_update_ibs_obs_cpc(dev); + result = 0; break; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 27039021c093..64e85426f084 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -236,6 +236,39 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo } #ifdef CONFIG_SOF_USERSPACE_LL + +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * move a part to the kernel thread: + * the userspace IPC handling thread would call comp_new_ipc4_user() to + * then call library manager .create method lib_manager_module_create(). + * That one calls lib_manager_mod_create_priv(), then + * lib_manager_allocate_module() and eventually + * llext_manager_allocate_module() for LLEXT modules. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace, &ops); +} + /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * From fd9f3be50418fff03dcfa65676a55a7e6f054e0d Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 16:14:08 +0200 Subject: [PATCH 26/30] schedule: ll: userspace: enable LLEXT LLEXT is now working with userspace LL and can be enabled. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 53b881320e6f..53c8763d6ce4 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -11,6 +11,7 @@ CONFIG_SOF_USERSPACE_LL=y # make the drivers work in user-space CONFIG_SOF_USERSPACE_INTERFACE_DMA=y CONFIG_DAI_USERSPACE=y +CONFIG_MAX_THREAD_BYTES=4 # Temporary settings that are needed currently to enable user-space LL # -------------------------------------------------------------------- @@ -24,11 +25,6 @@ CONFIG_COLD_STORE_EXECUTE_DEBUG=n CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n -# disable loadable modules (hits privilege issues in user-space now) -CONFIG_LLEXT_STORAGE_WRITABLE=n -CONFIG_LLEXT_EXPERIMENTAL=n -CONFIG_MODULES=n - # some of current boot tests interfere with user-space setup CONFIG_SOF_BOOT_TEST_ALLOWED=n From 7587d3b40e3098bffaed27f8ed7e21b18b0c25c8 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 19 Aug 2026 16:43:25 +0200 Subject: [PATCH 27/30] Revert "boards: intel: default to user-space LL for ptl and wcl" This reverts commit b647695127b6c23f4ebeb510f3cbd0c08a23dc24. --- app/boards/intel_adsp_ace30_ptl.conf | 28 ++++--------------- app/boards/intel_adsp_ace30_wcl.conf | 41 ++++------------------------ 2 files changed, 12 insertions(+), 57 deletions(-) diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 9a63751b1bde..b6ac41938398 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n -CONFIG_COLD_STORE_EXECUTE_DRAM=n +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y +CONFIG_COLD_STORE_EXECUTE_DRAM=y # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -40,10 +40,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=n -CONFIG_LLEXT_EXPERIMENTAL=n +CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_LLEXT_EDK=n -CONFIG_MODULES=n +CONFIG_MODULES=y # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -78,19 +78,3 @@ CONFIG_SOF_USERSPACE_PROXY=y CONFIG_MAX_THREAD_BYTES=3 CONFIG_MAX_DOMAIN_PARTITIONS=32 - -# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) -# Run Low-Latency audio pipelines in user-space threads by default. -CONFIG_SOF_USERSPACE_LL=y -CONFIG_SOF_USERSPACE_INTERFACE_DMA=y -CONFIG_DAI_USERSPACE=y - -# Settings currently required to enable user-space LL. The cold-store, -# telemetry, loadable-module and misc feature disables above/here are not -# yet user-space compatible (see the former overlay for rationale). -CONFIG_COLD_STORE_EXECUTE_DEBUG=n -CONFIG_SOF_BOOT_TEST_ALLOWED=n -CONFIG_CROSS_CORE_STREAM=n -CONFIG_INTEL_ADSP_MIC_PRIVACY=n -CONFIG_XRUN_NOTIFICATIONS_ENABLE=n -CONFIG_ZEPHYR_DP_SCHEDULER=n diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index d825a2a37c95..2196af333e65 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -21,9 +21,9 @@ CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL=n CONFIG_PROBE=y CONFIG_PROBE_DMA_MAX=2 CONFIG_SOF_TELEMETRY=y -CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n -CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n -CONFIG_COLD_STORE_EXECUTE_DRAM=n +CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=y +CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=y +CONFIG_COLD_STORE_EXECUTE_DRAM=y # SOF / loadable modules CONFIG_INTEL_MODULES=y @@ -39,10 +39,10 @@ CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y -CONFIG_LLEXT_STORAGE_WRITABLE=n -CONFIG_LLEXT_EXPERIMENTAL=n +CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_LLEXT_EDK=n -CONFIG_MODULES=n +CONFIG_MODULES=y # Zephyr / device drivers CONFIG_DAI_INIT_PRIORITY=70 @@ -64,32 +64,3 @@ CONFIG_PM_DEVICE_RUNTIME_ASYNC=n CONFIG_LOG_BACKEND_ADSP=n CONFIG_LOG_FLUSH_SLEEP_US=5000 CONFIG_WINSTREAM_CONSOLE=n - -# Userspace base (mirrored from intel_adsp_ace30_ptl.conf) -# Required so that user-space LL (below) can actually be enabled, since -# CONFIG_SOF_USERSPACE_LL depends on CONFIG_USERSPACE. -CONFIG_USERSPACE=y -CONFIG_DYNAMIC_THREAD=y -CONFIG_DYNAMIC_THREAD_ALLOC=y -CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y -CONFIG_SOF_STACK_SIZE=8192 -CONFIG_SOF_USERSPACE_PROXY=y -CONFIG_MAX_THREAD_BYTES=3 -CONFIG_MAX_DOMAIN_PARTITIONS=32 -CONFIG_XTENSA_MMU_NUM_L2_TABLES=128 - -# Userspace LL (was app/overlays/ptl/ll_userspace_overlay.conf) -# Run Low-Latency audio pipelines in user-space threads by default. -CONFIG_SOF_USERSPACE_LL=y -CONFIG_SOF_USERSPACE_INTERFACE_DMA=y -CONFIG_DAI_USERSPACE=y - -# Settings currently required to enable user-space LL. The cold-store, -# telemetry, loadable-module and misc feature disables above/here are not -# yet user-space compatible (see the former overlay for rationale). -CONFIG_COLD_STORE_EXECUTE_DEBUG=n -CONFIG_SOF_BOOT_TEST_ALLOWED=n -CONFIG_CROSS_CORE_STREAM=n -CONFIG_INTEL_ADSP_MIC_PRIVACY=n -CONFIG_XRUN_NOTIFICATIONS_ENABLE=n -CONFIG_ZEPHYR_DP_SCHEDULER=n From f22d116f416b4c8cb347115aac5e8afc11411f96 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:14:09 +0200 Subject: [PATCH 28/30] userspace-ll: re-enable DRAM execution and data CONFIG_COLD_STORE_EXECUTE_DRAM can now be re-enabled for userspace LL. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 53c8763d6ce4..4b89f63a8eb2 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -18,7 +18,6 @@ CONFIG_MAX_THREAD_BYTES=4 # problem with DSP panics due to illegal instruction hit in user-space if cold # store execution is enabled. Disable it for now until rootcause is found. -CONFIG_COLD_STORE_EXECUTE_DRAM=n CONFIG_COLD_STORE_EXECUTE_DEBUG=n # telemetry not yet user-space compatible From 62e97d11583b915a2b4c27d31314fe3982c3c1ed Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Thu, 25 Jun 2026 14:12:12 +0300 Subject: [PATCH 29/30] zephyr: lib: make vregion_alloc/free system calls Make vregion_alloc(), vregion_alloc_coherent(), vregion_alloc_align(), vregion_alloc_coherent_align(), and vregion_free() available as Zephyr system calls for user-space threads. Add K_SYSCALL_MEMORY_WRITE verification to all syscall handlers to validate the calling thread has access to the vregion's managed memory area. Add CONFIG_SOF_USERSPACE_INTERFACE_VREGION Kconfig option to control the feature. It is auto-selected by SOF_USERSPACE_LL when SOF_VREGIONS is enabled. Signed-off-by: Kai Vehmanen --- src/include/sof/lib/vregion.h | 26 ++++++++++--- zephyr/CMakeLists.txt | 2 + zephyr/Kconfig | 9 +++++ zephyr/lib/vregion.c | 25 ++++++------ zephyr/syscall/vregion.c | 73 +++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 zephyr/syscall/vregion.c diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 7d0bd11b27da..c403f6c08781 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -6,6 +6,8 @@ #define __SOF_LIB_VREGION_H__ #include +#include +#include #ifdef __cplusplus extern "C" { @@ -80,12 +82,16 @@ __syscall struct vregion *vregion_put(struct vregion *vr); * @param[in] size Size of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size); +__syscall void *vregion_alloc(struct vregion *vr, size_t size); + +void *z_impl_vregion_alloc(struct vregion *vr, size_t size); /** * @brief like vregion_alloc() but allocates coherent memory */ -void *vregion_alloc_coherent(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_coherent(struct vregion *vr, size_t size); + +void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size); /** * @brief Allocate aligned memory from the specified virtual region. @@ -98,12 +104,16 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size); * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); +__syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); + +void *z_impl_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief like vregion_alloc_align() but allocates coherent memory */ -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); +__syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); + +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); /** * @brief Free memory allocated from the specified virtual region. @@ -113,7 +123,9 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align * @param[in] vr Pointer to the virtual region instance. * @param[in] ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr); +__syscall void vregion_free(struct vregion *vr, void *ptr); + +void z_impl_vregion_free(struct vregion *vr, void *ptr); /** * @brief Log virtual region memory usage. @@ -183,4 +195,8 @@ static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t } #endif +#if CONFIG_SOF_VREGIONS +#include +#endif + #endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4f0813d582ed..636aa8f942d5 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -629,6 +629,8 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) +zephyr_library_sources(syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index ab88efcb9a6a..2d03a6d6f7ae 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -36,11 +36,20 @@ config SOF_USERSPACE_INTERFACE_ALLOC Allow user-space threads to use sof_heap_alloc/sof_heap_free as Zephyr system calls. +config SOF_USERSPACE_INTERFACE_VREGION + bool "Enable SOF vregion interface to userspace threads" + depends on USERSPACE + depends on SOF_VREGIONS + help + Allow user-space threads to use vregion_alloc/vregion_free + and their variants as Zephyr system calls. + config SOF_USERSPACE_LL bool "Run Low-Latency pipelines in userspace threads" depends on USERSPACE select SOF_USERSPACE_INTERFACE_ALLOC select SOF_USERSPACE_INTERFACE_DMA + select SOF_USERSPACE_INTERFACE_VREGION if SOF_VREGIONS help Run Low-Latency (LL) pipelines in userspace threads. This adds memory protection between operating system resources and diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 3eaee8e8e3b8..6a4813569c83 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -407,7 +407,7 @@ static void lifetime_free(struct vlinear_heap *heap, void *ptr) * @param vr Pointer to the virtual region instance. * @param ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr) +void z_impl_vregion_free(struct vregion *vr, void *ptr) { if (!vr || !ptr) return; @@ -432,7 +432,7 @@ void vregion_free(struct vregion *vr, void *ptr) k_mutex_unlock(&vr->lock); } -EXPORT_SYMBOL(vregion_free); +EXPORT_SYMBOL(z_impl_vregion_free); /** * @brief Allocate memory from the virtual region. @@ -443,7 +443,8 @@ EXPORT_SYMBOL(vregion_free); * * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_align(struct vregion *vr, + size_t size, size_t alignment) { void *p; @@ -471,7 +472,7 @@ void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) return p; } -EXPORT_SYMBOL(vregion_alloc_align); +EXPORT_SYMBOL(z_impl_vregion_alloc_align); /** * @brief Allocate memory from the virtual region. @@ -479,17 +480,17 @@ EXPORT_SYMBOL(vregion_alloc_align); * @param[in] size Size of the allocation. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size) +void *z_impl_vregion_alloc(struct vregion *vr, size_t size) { - return vregion_alloc_align(vr, size, 0); + return z_impl_vregion_alloc_align(vr, size, 0); } -EXPORT_SYMBOL(vregion_alloc); +EXPORT_SYMBOL(z_impl_vregion_alloc); -void *vregion_alloc_coherent(struct vregion *vr, size_t size) +void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size) { size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); + void *p = z_impl_vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); if (!p) return NULL; @@ -498,14 +499,15 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size) return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent); -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { if (alignment < CONFIG_DCACHE_LINE_SIZE) alignment = CONFIG_DCACHE_LINE_SIZE; size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, alignment); + void *p = z_impl_vregion_alloc_align(vr, size, alignment); if (!p) return NULL; @@ -514,6 +516,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent_align); /** * @brief Log virtual region memory usage. diff --git a/zephyr/syscall/vregion.c b/zephyr/syscall/vregion.c new file mode 100644 index 000000000000..70fb038eba05 --- /dev/null +++ b/zephyr/syscall/vregion.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +static inline void *z_vrfy_vregion_alloc(struct vregion *vr, size_t size) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc(vr, size); +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent(struct vregion *vr, size_t size) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc_coherent(vr, size); +} +#include + +static inline void *z_vrfy_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc_align(vr, size, alignment); +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent_align(struct vregion *vr, + size_t size, size_t alignment) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return z_impl_vregion_alloc_coherent_align(vr, size, alignment); +} +#include + +static inline void z_vrfy_vregion_free(struct vregion *vr, void *ptr) +{ + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + z_impl_vregion_free(vr, ptr); +} +#include From c2041bb953f84e92063f3380d617ff03d6a5a0e5 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 20 Aug 2026 15:18:17 +0200 Subject: [PATCH 30/30] userspace-ll: re-enable the DP scheduler The DP scheduler can now be user with userspace LL. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 4b89f63a8eb2..90082d26e16f 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -31,4 +31,3 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n -CONFIG_ZEPHYR_DP_SCHEDULER=n