[DNM][WiP][PoC] userspace LL scheduling: LLEXT & multicore

app/overlays/ptl/ll_userspace_overlay.conf

@@ -11,29 +11,23 @@ CONFIG_SOF_USERSPACE_LL=y
 # make the drivers work in user-space
 CONFIG_SOF_USERSPACE_INTERFACE_DMA=y
 CONFIG_DAI_USERSPACE=y
+CONFIG_MAX_THREAD_BYTES=4
 
 # Temporary settings that are needed currently to enable user-space LL
 # --------------------------------------------------------------------
 
 # problem with DSP panics due to illegal instruction hit in user-space if cold
 # store execution is enabled. Disable it for now until rootcause is found.
-CONFIG_COLD_STORE_EXECUTE_DRAM=n
 CONFIG_COLD_STORE_EXECUTE_DEBUG=n
 
 # telemetry not yet user-space compatible
 CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n
 CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n
 
-# disable loadable modules (hits privilege issues in user-space now)
-CONFIG_LLEXT_STORAGE_WRITABLE=n
-CONFIG_LLEXT_EXPERIMENTAL=n
-CONFIG_MODULES=n
-
 # some of current boot tests interfere with user-space setup
 CONFIG_SOF_BOOT_TEST_ALLOWED=n
 
 # misc features not yet compatible with user-space
 CONFIG_CROSS_CORE_STREAM=n
 CONFIG_INTEL_ADSP_MIC_PRIVACY=n
 CONFIG_XRUN_NOTIFICATIONS_ENABLE=n
-CONFIG_ZEPHYR_DP_SCHEDULER=n

src/audio/buffers/comp_buffer.c

@@ -11,6 +11,7 @@
 #include <sof/audio/sink_api.h>
 #include <sof/audio/source_api.h>
 #include <sof/audio/sink_source_utils.h>
+#include <sof/audio/module_adapter/module/generic.h>
 #include <rtos/userspace_helper.h>
 #include <sof/common.h>
 #include <rtos/interrupt.h>
@@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 
 	if (alloc && alloc->vreg) {
 		vregion_free(alloc->vreg, buffer);
-		if (!vregion_put(alloc->vreg))
-			rfree(alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(alloc ? alloc->heap : NULL, buffer);
 	}

src/audio/module_adapter/module_adapter.c

@@ -58,14 +58,80 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv,
 #define PAGE_SZ HOST_PAGE_SIZE
 #endif
 
-static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config,
-						  size_t *heap_size)
+struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config,
+					       uintptr_t *vreg_start, size_t *vreg_size)
 {
 	/* src-lite with 8 channels has been seen allocating 14k in one go */
 	/* FIXME: the size will be derived from configuration */
 	const size_t buf_size = 28 * 1024;
+	struct vregion *vr = vregion_create(buf_size);
 
-	return vregion_create(buf_size);
+	if (!vr)
+		return NULL;
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+	vregion_mem_info(vr, vreg_size, vreg_start);
+
+	/*
+	 * In the userspace LL case allocations are also performed by the
+	 * userspace IPC thread, which is also the one, executing this syscall
+	 */
+	struct k_mem_partition part = {
+		.start = *vreg_start,
+		.size = *vreg_size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB,
+	};
+	int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part);
+
+	if (ret < 0) {
+		vregion_put(vr);
+		return NULL;
+	}
+
+	part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start);
+	part.attr = K_MEM_PARTITION_P_RW_U_RW;
+
+	ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part);
+	if (ret < 0) {
+		vregion_put(vr);
+		return NULL;
+	}
+#else
+	ARG_UNUSED(vreg_start);
+	ARG_UNUSED(vreg_size);
+#endif
+
+	return vr;
+}
+
+void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc)
+{
+#ifdef CONFIG_SOF_USERSPACE_LL
+	struct k_mem_partition part = {
+		.start = alloc->vreg_start,
+		.size = alloc->vreg_size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB,
+	};
+
+	k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part);
+
+	part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start);
+	part.attr = K_MEM_PARTITION_P_RW_U_RW;
+
+	k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part);
+#else
+	ARG_UNUSED(alloc);
+#endif
+}
+
+void module_adapter_vreg_free(struct mod_alloc_ctx *alloc)
+{
+	if (vregion_put(alloc->vreg))
+		return;
+
+	module_adapter_vreg_unmap(alloc);
+
+	sof_heap_free(alloc->heap, alloc);
 }
 
 static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv,
@@ -84,11 +150,12 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	 */
 	uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ?
 		SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER;
-	size_t heap_size;
+	size_t vreg_size;
+	uintptr_t vreg_start;
 
 	if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) &&
 	    IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) {
-		mod_vreg = module_adapter_dp_heap_new(config, &heap_size);
+		mod_vreg = module_adapter_vreg_new(config, &vreg_start, &vreg_size);
 		if (!mod_vreg) {
 			comp_cl_err(drv, "Failed to allocate DP module heap / vregion");
 			return NULL;
@@ -105,7 +172,6 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 #else
 		mod_heap = drv->user_heap;
 #endif
-		heap_size = 0;
 		mod_vreg = NULL;
 	}
 
@@ -129,6 +195,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	memset(mod, 0, sizeof(*mod));
 	alloc->heap = mod_heap;
 	alloc->vreg = mod_vreg;
+	alloc->vreg_start = vreg_start;
+	alloc->vreg_size = vreg_size;
 	mod->priv.resources.alloc = alloc;
 	mod_resource_init(mod);
 
@@ -169,6 +237,25 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	return NULL;
 }
 
+#ifdef CONFIG_USERSPACE
+#include <zephyr/internal/syscall_handler.h>
+struct vregion *z_vrfy_module_adapter_vreg_new(const struct comp_ipc_config *config,
+					       uintptr_t *vreg_start, size_t *vreg_size)
+{
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start)));
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size)));
+	K_OOPS(K_SYSCALL_MEMORY_READ(config, sizeof(*config)));
+	return z_impl_module_adapter_vreg_new(config, vreg_start, vreg_size);
+}
+#include <zephyr/syscalls/module_adapter_vreg_new_mrsh.c>
+void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc)
+{
+	K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc)));
+	z_impl_module_adapter_vreg_unmap(alloc);
+}
+#include <zephyr/syscalls/module_adapter_vreg_unmap_mrsh.c>
+#endif
+
 static void module_adapter_mem_free(struct processing_module *mod)
 {
 	struct mod_alloc_ctx *alloc = mod->priv.resources.alloc;
@@ -186,8 +273,7 @@ static void module_adapter_mem_free(struct processing_module *mod)
 
 		vregion_free(mod_vreg, mod->dev);
 		vregion_free(mod_vreg, mod);
-		if (!vregion_put(mod_vreg))
-			sof_heap_free(alloc->heap, alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(mod_heap, mod->dev);
 		sof_heap_free(mod_heap, mod);

src/audio/pipeline/pipeline-graph.c

@@ -15,10 +15,12 @@
 #include <rtos/interrupt.h>
 #include <rtos/symbol.h>
 #include <rtos/alloc.h>
+#include <rtos/userspace_helper.h>
 #include <sof/lib/mm_heap.h>
 #include <sof/lib/uuid.h>
 #include <sof/compiler_attributes.h>
 #include <sof/list.h>
+#include <sof/schedule/ll_schedule_domain.h>
 #include <rtos/spinlock.h>
 #include <rtos/string.h>
 #include <rtos/clk.h>
@@ -44,19 +46,68 @@ DECLARE_TR_CTX(pipe_tr, SOF_UUID(pipe_uuid), LOG_LEVEL_INFO);
 /* lookup table to determine busy/free pipeline metadata objects */
 struct pipeline_posn {
 	bool posn_offset[PPL_POSN_OFFSETS];	/**< available offsets */
+#ifndef CONFIG_SOF_USERSPACE_LL
 	struct k_spinlock lock;			/**< lock mechanism */
+#endif
 };
 /* the pipeline position lookup table */
-static SHARED_DATA struct pipeline_posn pipeline_posn_shared;
+static APP_SYSUSER_BSS SHARED_DATA struct pipeline_posn pipeline_posn_shared;
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+/* Mutex pointer in user-accessible partition so user-space threads
+ * can read the pointer for syscalls. Kept outside the SHARED_DATA
+ * struct to avoid kernel object tracking issues.
+ */
+static APP_SYSUSER_BSS struct k_mutex *pipeline_posn_mutex;
+#endif
 
 /**
  * \brief Retrieves pipeline position structure.
  * \return Pointer to pipeline position structure.
  */
 static inline struct pipeline_posn *pipeline_posn_get(void)
 {
+#ifdef CONFIG_SOF_USERSPACE_LL
+	return &pipeline_posn_shared;
+#else
 	return sof_get()->pipeline_posn;
+#endif
+}
+
+/*
+ * Position table locking. User-space LL cannot use a spinlock (disabling
+ * interrupts is privileged), so it uses a mutex; the config split is kept
+ * here so the callers below stay identical for both configurations.
+ */
+#ifdef CONFIG_SOF_USERSPACE_LL
+typedef int pipeline_posn_key_t;
+
+static inline pipeline_posn_key_t pipeline_posn_lock(struct pipeline_posn *posn)
+{
+	(void)posn;
+	k_mutex_lock(pipeline_posn_mutex, K_FOREVER);
+	return 0;
+}
+
+static inline void pipeline_posn_unlock(struct pipeline_posn *posn, pipeline_posn_key_t key)
+{
+	(void)posn;
+	(void)key;
+	k_mutex_unlock(pipeline_posn_mutex);
+}
+#else
+typedef k_spinlock_key_t pipeline_posn_key_t;
+
+static inline pipeline_posn_key_t pipeline_posn_lock(struct pipeline_posn *posn)
+{
+	return k_spin_lock(&posn->lock);
+}
+
+static inline void pipeline_posn_unlock(struct pipeline_posn *posn, pipeline_posn_key_t key)
+{
+	k_spin_unlock(&posn->lock, key);
 }
+#endif
 
 /**
  * \brief Retrieves first free pipeline position offset.
@@ -68,9 +119,7 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset)
 	struct pipeline_posn *pipeline_posn = pipeline_posn_get();
 	int ret = -EINVAL;
 	uint32_t i;
-	k_spinlock_key_t key;
-
-	key = k_spin_lock(&pipeline_posn->lock);
+	pipeline_posn_key_t key = pipeline_posn_lock(pipeline_posn);
 
 	for (i = 0; i < PPL_POSN_OFFSETS; ++i) {
 		if (!pipeline_posn->posn_offset[i]) {
@@ -81,8 +130,7 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset)
 		}
 	}
 
-
-	k_spin_unlock(&pipeline_posn->lock, key);
+	pipeline_posn_unlock(pipeline_posn, key);
 
 	return ret;
 }
@@ -95,21 +143,35 @@ static inline void pipeline_posn_offset_put(uint32_t posn_offset)
 {
 	struct pipeline_posn *pipeline_posn = pipeline_posn_get();
 	int i = posn_offset / sizeof(struct sof_ipc_stream_posn);
-	k_spinlock_key_t key;
-
-	key = k_spin_lock(&pipeline_posn->lock);
+	pipeline_posn_key_t key = pipeline_posn_lock(pipeline_posn);
 
 	pipeline_posn->posn_offset[i] = false;
 
-	k_spin_unlock(&pipeline_posn->lock, key);
+	pipeline_posn_unlock(pipeline_posn, key);
 }
 
 void pipeline_posn_init(struct sof *sof)
 {
 	sof->pipeline_posn = &pipeline_posn_shared;
+#ifdef CONFIG_SOF_USERSPACE_LL
+	pipeline_posn_mutex = k_object_alloc(K_OBJ_MUTEX);
+	if (!pipeline_posn_mutex) {
+		pipe_cl_err("pipeline posn mutex alloc failed");
+		k_panic();
+	}
+	k_mutex_init(pipeline_posn_mutex);
+#else
 	k_spinlock_init(&sof->pipeline_posn->lock);
+#endif
 }
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+void pipeline_posn_grant_access(struct k_thread *thread)
+{
+	k_thread_access_grant(thread, pipeline_posn_mutex);
+}
+#endif
+
 /* create new pipeline - returns pipeline id or negative error */
 struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority,
 			      uint32_t comp_id, struct create_pipeline_params *pparams)
@@ -140,12 +202,21 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_
 	p->pipeline_id = pipeline_id;
 	p->status = COMP_STATE_INIT;
 	p->trigger.cmd = COMP_TRIGGER_NO_ACTION;
+
+#ifndef CONFIG_SOF_USERSPACE_LL
+	/*
+	 * pipe_tr lives in the .trace_ctx section, which is not mapped into
+	 * the sysuser partition, so it cannot be read from a user-mode thread.
+	 * The copy is also unnecessary in that configuration: with Zephyr
+	 * logging the pipe_*() macros use the global pipe_tr, not p->tctx.
+	 */
 	ret = memcpy_s(&p->tctx, sizeof(struct tr_ctx), &pipe_tr,
 		       sizeof(struct tr_ctx));
 	if (ret < 0) {
 		pipe_err(p, "failed to copy trace settings");
 		goto free;
 	}
+#endif
 
 	ret = pipeline_posn_offset_get(&p->posn_offset);
 	if (ret < 0) {

src/include/ipc4/handler.h

@@ -16,6 +16,14 @@ struct ipc4_message_request;
  */
 int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply);
 
+/**
+ * \brief Load a dynamically loadable module.
+ * @param[in] drv Component driver.
+ * @param[in] mi SOF_IPC4_MOD_INIT_INSTANCE data
+ */
+int ipc4_user_module_load(const struct comp_driver *drv,
+			  const struct ipc4_module_init_instance *mi);
+
 /**
  * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context.
  * @param[in] ipc4 IPC4 message request.