diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 48a923ab7870..7d1ade754607 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -38,7 +38,7 @@ CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 -CONFIG_LLEXT=y +CONFIG_LLEXT=n CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y @@ -65,3 +65,6 @@ CONFIG_LOG_BACKEND_ADSP=n CONFIG_LOG_FLUSH_SLEEP_US=5000 CONFIG_LOG_OUTPUT_FORMAT_LINUX_TIMESTAMP=y CONFIG_WINSTREAM_CONSOLE=n + +CONFIG_USERSPACE=y +CONFIG_SOF_USERSPACE_PROXY=y \ No newline at end of file diff --git a/src/audio/module_adapter/CMakeLists.txt b/src/audio/module_adapter/CMakeLists.txt index ab15558d361d..e341e90d872c 100644 --- a/src/audio/module_adapter/CMakeLists.txt +++ b/src/audio/module_adapter/CMakeLists.txt @@ -57,6 +57,10 @@ if(zephyr) ### Zephyr ### library/native_system_service.c ) + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_PROXY + library/userspace_proxy.c + ) + zephyr_library_sources_ifdef(CONFIG_PASSTHROUGH_CODEC module/passthrough.c ) diff --git a/src/audio/module_adapter/iadk/system_agent.cpp b/src/audio/module_adapter/iadk/system_agent.cpp index 860d5b1b279b..caed8cf65860 100644 --- a/src/audio/module_adapter/iadk/system_agent.cpp +++ b/src/audio/module_adapter/iadk/system_agent.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include using namespace intel_adsp; using namespace intel_adsp::system; @@ -37,7 +39,7 @@ namespace system { /* Structure storing handles to system service operations */ -const AdspSystemService SystemAgent::system_service_ = { +const APP_TASK_DATA AdspSystemService SystemAgent::system_service_ = { native_system_service_log_message, native_system_service_safe_memcpy, native_system_service_safe_memmove, @@ -124,16 +126,17 @@ int SystemAgent::CheckIn(ProcessingModuleFactoryInterface& module_factory, typedef int (*create_instance_f)(uint32_t module_id, uint32_t instance_id, uint32_t core_id, void *mod_cfg, void *parent_ppl, void **mod_ptr); -int system_agent_start(uintptr_t entry_point, uint32_t module_id, uint32_t instance_id, - uint32_t core_id, uint32_t log_handle, void* mod_cfg, +int system_agent_start(const struct system_agent_params *params, const void **adapter) { uint32_t ret; - SystemAgent system_agent(module_id, instance_id, core_id, log_handle); + SystemAgent system_agent(params->module_id, params->instance_id, params->core_id, + params->log_handle); void* system_agent_p = reinterpret_cast(&system_agent); - create_instance_f ci = (create_instance_f)(entry_point); - ret = ci(module_id, instance_id, core_id, mod_cfg, NULL, &system_agent_p); + create_instance_f ci = (create_instance_f)(params->entry_point); + ret = ci(params->module_id, params->instance_id, params->core_id, params->mod_cfg, NULL, + &system_agent_p); IadkModuleAdapter* module_adapter = reinterpret_cast(system_agent_p); *adapter = module_adapter; diff --git a/src/audio/module_adapter/library/native_system_agent.c b/src/audio/module_adapter/library/native_system_agent.c index 9a93af6f3565..b93405b8bd27 100644 --- a/src/audio/module_adapter/library/native_system_agent.c +++ b/src/audio/module_adapter/library/native_system_agent.c @@ -19,21 +19,20 @@ typedef void* (*native_create_instance_f)(void *mod_cfg, void *parent_ppl, struct native_system_agent native_sys_agent; -int native_system_agent_start(uintptr_t entry_point, uint32_t module_id, uint32_t instance_id, - uint32_t core_id, uint32_t log_handle, void *mod_cfg, +int native_system_agent_start(const struct system_agent_params *params, const void **iface) { - native_sys_agent.module_id = module_id; - native_sys_agent.instance_id = instance_id; - native_sys_agent.core_id = core_id; - native_sys_agent.log_handle = log_handle; + native_sys_agent.module_id = params->module_id; + native_sys_agent.instance_id = params->instance_id; + native_sys_agent.core_id = params->core_id; + native_sys_agent.log_handle = params->log_handle; const void *ret; void *system_agent_p = &native_sys_agent; - native_create_instance_f ci = (native_create_instance_f)entry_point; + native_create_instance_f ci = (native_create_instance_f)params->entry_point; - ret = ci(mod_cfg, NULL, &system_agent_p); + ret = ci(params->mod_cfg, NULL, &system_agent_p); if (!ret) return -EINVAL; diff --git a/src/audio/module_adapter/library/native_system_service.c b/src/audio/module_adapter/library/native_system_service.c index 5126fa5ff12b..1c4688e885d0 100644 --- a/src/audio/module_adapter/library/native_system_service.c +++ b/src/audio/module_adapter/library/native_system_service.c @@ -19,6 +19,7 @@ #include #include #include +#include #define RSIZE_MAX 0x7FFFFFFF @@ -162,7 +163,7 @@ AdspErrorCode native_system_service_get_interface(enum interface_id id, return ADSP_NO_ERROR; } -const struct native_system_service native_system_service = { +const APP_TASK_DATA struct native_system_service native_system_service = { .basic = { .log_message = native_system_service_log_message, .safe_memcpy = native_system_service_safe_memcpy, diff --git a/src/audio/module_adapter/library/userspace_proxy.c b/src/audio/module_adapter/library/userspace_proxy.c new file mode 100644 index 000000000000..fffae4ad73a6 --- /dev/null +++ b/src/audio/module_adapter/library/userspace_proxy.c @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2023 Intel Corporation. All rights reserved. +// +// Author: Jaroslaw Stelter +// Author: Adrian Warecki + +/** + * \file audio/module_adapter/library/userspace_proxy.c + * \brief Userspace proxy. Acts as an intermediary between SOF and a userspace module. + * \brief Responsible for preparing the memory domain required for userspace execution + * \brief and forwarding API calls. The proxy invokes corresponding module methods + * \brief in userspace context. Enables execution of any module implementing module_interface + * \brief as a userspace module. + * \authors Adrian Warecki + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +LOG_MODULE_REGISTER(userspace_proxy, CONFIG_SOF_LOG_LEVEL); + +/* 6f6b6f4b-6f73-7466-20e1e62b9779f003 */ +SOF_DEFINE_REG_UUID(userspace_proxy); + +DECLARE_TR_CTX(userspace_proxy_tr, SOF_UUID(userspace_proxy_uuid), LOG_LEVEL_INFO); + +static const struct module_interface userspace_proxy_interface; + +static int userspace_proxy_memory_init(struct userspace_context *user, + const struct comp_driver *drv) +{ + /* Add module private heap to memory partitions */ + struct k_mem_partition heap_part = { .attr = K_MEM_PARTITION_P_RW_U_RW }; + + k_mem_region_align(&heap_part.start, &heap_part.size, + POINTER_TO_UINT(drv->user_heap->init_mem), + drv->user_heap->init_bytes, CONFIG_MM_DRV_PAGE_SIZE); + + tr_err(&userspace_proxy_tr, "Heap partition %p + %#zx, attr = %u", + UINT_TO_POINTER(heap_part.start), heap_part.size, heap_part.attr); + +#if !defined(CONFIG_XTENSA_MMU_DOUBLE_MAP) && defined(CONFIG_SOF_ZEPHYR_HEAP_CACHED) +#define HEAP_PART_CACHED + /* Add cached module private heap to memory partitions */ + struct k_mem_partition heap_cached_part = { .attr = K_MEM_PARTITION_P_RW_U_RW }; + + k_mem_region_align(&heap_cached_part.start, &heap_cached_part.size, + POINTER_TO_UINT(sys_cache_cached_ptr_get(drv->user_heap->init_mem)), + drv->user_heap->init_bytes, CONFIG_MM_DRV_PAGE_SIZE); + + tr_err(&userspace_proxy_tr, "Cached heap partition %p + %#zx, attr = %u", + UINT_TO_POINTER(heap_cached_part.start), heap_cached_part.size, + heap_cached_part.attr); +#endif + + struct k_mem_partition *parts_ptr[] = { +#ifdef HEAP_PART_CACHED + &heap_part_cached, +#endif + &heap_part + }; + + return k_mem_domain_init(user->comp_dom, ARRAY_SIZE(parts_ptr), parts_ptr); +} + +static int userspace_proxy_add_sections(struct userspace_context *user, uint32_t instance_id, + const struct sof_man_module *const mod) +{ + struct k_mem_partition mem_partition; + void *va_base; + int idx, ret; + + for (idx = 0; idx < ARRAY_SIZE(mod->segment); ++idx) { + if (!mod->segment[idx].flags.r.load) + continue; + + if (mod->segment[idx].flags.r.code) + mem_partition.attr = K_MEM_PARTITION_P_RX_U_RX; + else if (!mod->segment[idx].flags.r.readonly) + mem_partition.attr = K_MEM_PARTITION_P_RW_U_RW; + else + mem_partition.attr = K_MEM_PARTITION_P_RO_U_RO; + + mem_partition.start = mod->segment[idx].v_base_addr; + mem_partition.size = mod->segment[idx].flags.r.length * CONFIG_MM_DRV_PAGE_SIZE; + + ret = k_mem_domain_add_partition(user->comp_dom, &mem_partition); + + tr_err(&userspace_proxy_tr, "Add mod partition %p + %#zx, attr = %u, ret = %d", + UINT_TO_POINTER(mem_partition.start), mem_partition.size, + mem_partition.attr, ret); + + if (ret < 0) + return ret; + } + + lib_manager_get_instance_bss_address(instance_id, mod, &va_base, &mem_partition.size); + mem_partition.start = POINTER_TO_UINT(va_base); + mem_partition.attr = K_MEM_PARTITION_P_RW_U_RW; + ret = k_mem_domain_add_partition(user->comp_dom, &mem_partition); + + tr_err(&userspace_proxy_tr, "Add bss partition %p + %#zx, attr = %u, ret = %d", + UINT_TO_POINTER(mem_partition.start), mem_partition.size, + mem_partition.attr, ret); + + return ret; +} + +int userspace_proxy_create(struct userspace_context **user_ctx, const struct comp_driver *drv, + const struct sof_man_module *manifest, system_agent_start_fn start_fn, + const struct system_agent_params *agent_params, + const void **agent_interface, const struct module_interface **ops) +{ + struct userspace_context *user; + struct k_mem_domain *domain; + int ret; + + tr_err(&userspace_proxy_tr, "userspace create"); + + user = sys_heap_alloc(drv->user_heap, sizeof(struct userspace_context)); + if (!user) + return -ENOMEM; + + /* Allocate memory domain struct */ + domain = rzalloc(SOF_MEM_FLAG_KERNEL, sizeof(*domain)); + if (!domain) { + ret = -ENOMEM; + goto error; + } + user->comp_dom = domain; + + ret = userspace_proxy_memory_init(user, drv); + if (ret) + goto error_dom; + + ret = userspace_proxy_add_sections(user, agent_params->instance_id, manifest); + if (ret) + goto error_dom; + + /* Start the system agent, if provided. */ + if (start_fn) { + ret = start_fn(agent_params, agent_interface); + + if (ret) { + tr_err(&userspace_proxy_tr, "System agent failed with error %d.", ret); + goto error_dom; + } + } + + *user_ctx = user; + + /* Store a pointer to the module's interface. For the LMDK modules, the agent places a + * pointer to the module interface at the address specified by agent_interface. Since this + * points to ops, the assignment of the module interface used by this proxy must occur + * after the agent has been started. For other module types, the ops parameter points to a + * valid module interface. + */ + user->interface = *ops; + + /* All calls to the module interface must pass through the proxy. Set up our own interface. + */ + *ops = &userspace_proxy_interface; + + return 0; + +error_dom: + rfree(domain); +error: + sys_heap_free(drv->user_heap, user); + return ret; +} + +void userspace_proxy_destroy(const struct comp_driver *drv, struct userspace_context *user_ctx) +{ + tr_err(&userspace_proxy_tr, "userspace proxy destroy"); + rfree(user_ctx->comp_dom); + sys_heap_free(drv->user_heap, user_ctx); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module init() operation and return its result. + * Module init() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_init(struct processing_module *mod) +{ + comp_dbg(mod->dev, "start"); + + /* Return status from module code operation. */ + return mod->user_ctx->interface->init(mod); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module prepare() operation and return its result. + * Module prepare() code is performed in user workqueue. + * + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->prepare) + return 0; + + return mod->user_ctx->interface->prepare(mod, sources, num_of_sources, sinks, num_of_sinks); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module init() operation and return its result. + * Module init() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_process(struct processing_module *mod, struct sof_source **sources, + int num_of_sources, struct sof_sink **sinks, int num_of_sinks) +{ + return mod->user_ctx->interface->process(mod, sources, num_of_sources, sinks, num_of_sinks); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module reset() operation and return its result. + * Module reset() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_reset(struct processing_module *mod) +{ + if (!mod->user_ctx->interface->reset) + return 0; + + return mod->user_ctx->interface->reset(mod); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module free() operation and return its result. + * Module free() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_free(struct processing_module *mod) +{ + int ret = 0; + + comp_dbg(mod->dev, "start"); + + if (mod->user_ctx->interface->free) + ret = mod->user_ctx->interface->free(mod); + + userspace_proxy_destroy(mod->dev->drv, mod->user_ctx); + mod->user_ctx = NULL; + + /* Return status from module code operation. */ + return ret; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module set_configuration() operation and return its result. + * Module set_configuration() code is performed in user workqueue. + * + * @param[in] mod - struct processing_module pointer + * @param[in] config_id - Configuration ID + * @param[in] pos - position of the fragment in the large message + * @param[in] data_offset_size: size of the whole configuration if it is the first fragment or the + * only fragment. Otherwise, it is the offset of the fragment in the whole + * configuration. + * @param[in] fragment: configuration fragment buffer + * @param[in] fragment_size: size of @fragment + * @params[in] response: optional response buffer to fill + * @params[in] response_size: size of @response + * + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_set_configuration(struct processing_module *mod, uint32_t config_id, + enum module_cfg_fragment_position pos, + uint32_t data_offset_size, const uint8_t *fragment, + size_t fragment_size, uint8_t *response, + size_t response_size) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->set_configuration) + return 0; + + return mod->user_ctx->interface->set_configuration(mod, config_id, pos, data_offset_size, + fragment, fragment_size, + response, response_size); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module get_configuration() operation and return its result. + * Module get_configuration() code is performed in user workqueue. + * + * @param[in] mod - struct processing_module pointer + * @param[in] config_id - Configuration ID + * @param[in] data_offset_size: size of the whole configuration if it is the first fragment or the + * only fragment. Otherwise, it is the offset of the fragment in the whole + * configuration. + * @param[in] fragment: configuration fragment buffer + * @param[in] fragment_size: size of @fragment + * + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_get_configuration(struct processing_module *mod, uint32_t config_id, + uint32_t *data_offset_size, uint8_t *fragment, + size_t fragment_size) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->get_configuration) + return -EIO; + + return mod->user_ctx->interface->get_configuration(mod, config_id, data_offset_size, + fragment, fragment_size); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module set_processing_mode() operation and return its result. + * Module set_processing_mode() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @param mode - processing mode to be set. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_set_processing_mode(struct processing_module *mod, + enum module_processing_mode mode) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->set_processing_mode) + return 0; + + return mod->user_ctx->interface->set_processing_mode(mod, mode); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module get_processing_mode() operation and return its result. + * Module get_processing_mode() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return processing mode. + */ +static +enum module_processing_mode userspace_proxy_get_processing_mode(struct processing_module *mod) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->get_processing_mode) + return -EIO; + + return mod->user_ctx->interface->get_processing_mode(mod); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module is_ready_to_process() operation and return its result. + * Module is_ready_to_process() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return true if the module is ready to process + */ +static bool userspace_proxy_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, + int num_of_sources, + struct sof_sink **sinks, + int num_of_sinks) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->is_ready_to_process) + return generic_module_is_ready_to_process(mod, sources, num_of_sources, sinks, + num_of_sinks); + + return mod->user_ctx->interface->is_ready_to_process(mod, sources, num_of_sources, + sinks, num_of_sinks); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module bind() operation and return its result. + * Module bind() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @param bind_data - pointer to bind_info structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_bind(struct processing_module *mod, struct bind_info *bind_data) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->bind) + return 0; + + return mod->user_ctx->interface->bind(mod, bind_data); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module unbind() operation and return its result. + * Module unbind() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @param unbind_data - pointer to bind_info structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_unbind(struct processing_module *mod, struct bind_info *unbind_data) +{ + comp_dbg(mod->dev, "start"); + + if (!mod->user_ctx->interface->unbind) + return 0; + + return mod->user_ctx->interface->unbind(mod, unbind_data); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module trigger() operation and return its result. + * Module trigger() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_proxy_trigger(struct processing_module *mod, int cmd) +{ + int ret = 0; + + comp_dbg(mod->dev, "start"); + + if (mod->user_ctx->interface->trigger) + ret = mod->user_ctx->interface->trigger(mod, cmd); + + if (!ret) + ret = module_adapter_set_state(mod, mod->dev, cmd); + + /* Return status from module code operation. */ + return ret; +} + +/* Userspace Module Adapter API */ +APP_TASK_DATA static const struct module_interface userspace_proxy_interface = { + .init = userspace_proxy_init, + .is_ready_to_process = userspace_proxy_is_ready_to_process, + .prepare = userspace_proxy_prepare, + .process = userspace_proxy_process, + .set_configuration = userspace_proxy_set_configuration, + .get_configuration = userspace_proxy_get_configuration, + .set_processing_mode = userspace_proxy_set_processing_mode, + .get_processing_mode = userspace_proxy_get_processing_mode, + .reset = userspace_proxy_reset, + .free = userspace_proxy_free, + .bind = userspace_proxy_bind, + .unbind = userspace_proxy_unbind, + .trigger = userspace_proxy_trigger, +}; diff --git a/src/audio/module_adapter/module/modules.c b/src/audio/module_adapter/module/modules.c index 3bb34c449478..66f257f62c11 100644 --- a/src/audio/module_adapter/module/modules.c +++ b/src/audio/module_adapter/module/modules.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -201,7 +202,7 @@ static int modules_reset(struct processing_module *mod) } /* Processing Module Adapter API*/ -const struct module_interface processing_module_adapter_interface = { +APP_TASK_DATA const struct module_interface processing_module_adapter_interface = { .init = modules_init, .prepare = modules_prepare, .process = modules_process, diff --git a/src/audio/module_adapter/module/modules_userspace.c b/src/audio/module_adapter/module/modules_userspace.c new file mode 100644 index 000000000000..29ff0030ad8f --- /dev/null +++ b/src/audio/module_adapter/module/modules_userspace.c @@ -0,0 +1,1040 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2023 Intel Corporation. All rights reserved. +// +// Author: Jaroslaw Stelter +// Author: Adrian Warecki +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + + +LOG_MODULE_REGISTER(modules_user_tr, CONFIG_SOF_LOG_LEVEL); + +/* 7c42ce8b-0108-43d0-9137-56d660478c55 */ +SOF_DEFINE_REG_UUID(modules_user); + +DECLARE_TR_CTX(modules_user_tr, SOF_UUID(modules_user_uuid), LOG_LEVEL_INFO); + +/* IPC works synchronously so single message queue is ok */ +#define MSGQ_LEN 1 + +K_APPMEM_PARTITION_DEFINE(ipc_partition); +#define MAX_PARAM_SIZE 0x200 + +struct user_worker_data { + struct k_msgq *ipc_in_msg_q; /* pointer to input message queue */ + struct k_msgq *ipc_out_msg_q; /* pointer to output message queue */ + struct k_work_user work_item; /* ipc worker workitem */ + k_tid_t ipc_worker_tid; /* ipc worker thread ID */ + uint8_t ipc_params[MAX_PARAM_SIZE]; /* ipc parameter buffer */ + uint32_t module_ref_cnt; /* module reference count */ + void *p_worker_stack; /* pointer to worker stack */ +}; + +struct user_security_domain { + struct k_work_user_q ipc_user_work_q; + struct k_msgq in_msgq; + struct k_msgq out_msgq; +}; + +static struct user_security_domain security_domain[CONFIG_SEC_DOMAIN_MAX_NUMBER]; +/* + * There is one instance of IPC user worker per security domain. + * Keep pointer to it here + */ +APP_USER_BSS static struct user_worker_data *worker_data[CONFIG_SEC_DOMAIN_MAX_NUMBER]; + +struct module_agent_params { + system_agent_start_fn start_fn; + uintptr_t entry_point; + uint32_t module_id; + uint32_t instance_id; + uint32_t core_id; + uint32_t log_handle; + byte_array_t mod_cfg; + void *iface; +}; + +struct module_large_cfg_set_params { + uint32_t config_id; + enum module_cfg_fragment_position pos; + uint32_t data_offset_size; + const uint8_t *fragment; + size_t fragment_size; + uint8_t *response; + size_t response_size; +}; + +struct module_large_cfg_get_params { + uint32_t config_id; + uint32_t *data_offset_size; + uint8_t *fragment; + size_t fragment_size; +}; + +struct module_processing_mode_params { + enum module_processing_mode mode; +}; + +struct module_process_params { + struct sof_source **sources; + int num_of_sources; + struct sof_sink **sinks; + int num_of_sinks; +}; + +struct module_params { + uint32_t cmd; + int status; + struct processing_module *mod; + struct userspace_module *user; + union { + struct module_agent_params agent; + struct module_large_cfg_set_params set_conf; + struct module_large_cfg_get_params get_conf; + struct module_processing_mode_params proc_mode; + struct module_process_params proc; + void *bind_data; + int trigger_data; + } ext; +}; + +static const struct module_interface userspace_module_adapter_interface; + +enum { + MODULE_CMD_AGENT_START, + MODULE_CMD_INIT, + MODULE_CMD_PREPARE, + MODULE_CMD_PROC_READY, + MODULE_CMD_PROCESS, + MODULE_CMD_SET_PROCMOD, + MODULE_CMD_GET_PROCMOD, + MODULE_CMD_SET_CONF, + MODULE_CMD_GET_CONF, + MODULE_CMD_BIND, + MODULE_CMD_UNBIND, + MODULE_CMD_RESET, + MODULE_CMD_FREE, + MODULE_CMD_TRIGGER +}; + +static int modules_msgq_alloc(struct user_security_domain *security_domain, + struct user_worker_data *worker_data) +{ + char *buffer; + + buffer = rzalloc(SOF_MEM_ZONE_SYS_RUNTIME, 0, SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_MMU_SHD, + MAX_PARAM_SIZE * 2); + if (!buffer) + return -ENOMEM; + + k_msgq_init(&security_domain->in_msgq, buffer, MAX_PARAM_SIZE, 1); + k_msgq_init(&security_domain->out_msgq, (buffer + MAX_PARAM_SIZE), MAX_PARAM_SIZE, 1); + + worker_data->ipc_in_msg_q = &security_domain->in_msgq; + worker_data->ipc_out_msg_q = &security_domain->out_msgq; + + return 0; +} + +static void modules_msgq_free(struct user_worker_data *worker_data) +{ + /* ipc_in_msg_q->buffer_start points to whole buffer allocated for message queues */ + rfree(worker_data->ipc_in_msg_q->buffer_start); +} +static int modules_user_init_worker(struct userspace_module *user, + k_work_user_handler_t user_handler) +{ + uint32_t sd_id = user->security_domain_id; + struct user_worker_data *wd = worker_data[sd_id]; + struct user_security_domain *sd = &security_domain[sd_id]; + int ret; + tr_dbg(&modules_user_tr, "start"); + + if (sd_id >= CONFIG_SEC_DOMAIN_MAX_NUMBER) { + tr_err(&modules_user_tr, "Invalid security_domain_id"); + return -EINVAL; + } + + /* If worker_data not initialized this is first userspace module in security domain */ + if (!wd) { + wd = rzalloc(SOF_MEM_ZONE_SYS_RUNTIME, 0, SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_MMU_SHD, + sizeof(*wd)); + if (!wd) + return -ENOMEM; + + /* Store worker data in processing module structure */ + worker_data[sd_id] = wd; + user->wrk_ctx = wd; + + ret = modules_msgq_alloc(sd, wd); + if (ret < 0) + goto err_worker; + + wd->p_worker_stack = user_stack_allocate(CONFIG_SOF_STACK_SIZE, K_USER); + if (!wd->p_worker_stack) { + tr_err(&modules_user_tr, "stack alloc failed"); + ret = -ENOMEM; + goto err_msgq; + } + + /* Create User Mode work queue start before submit. */ + k_work_user_queue_start(&sd->ipc_user_work_q, wd->p_worker_stack, + CONFIG_SOF_STACK_SIZE, 0, NULL); + + wd->ipc_worker_tid = + k_work_user_queue_thread_get(&sd->ipc_user_work_q); + } + + /* Init module memory domain. */ + ret = k_mem_domain_add_thread(user->comp_dom, wd->ipc_worker_tid); + if (ret < 0) { + tr_err(&modules_user_tr, "failed to add memory domain: error: %d", ret); + goto err_init; + } + + if (!wd->module_ref_cnt) { + k_work_user_init(&wd->work_item, user_handler); + + /* Grant a thread access to a kernel object (IPC msg. data). */ + k_thread_access_grant(wd->ipc_worker_tid, + wd->ipc_in_msg_q, + wd->ipc_out_msg_q); + + + + /* Submit work item to the queue */ + ret = k_work_user_submit_to_queue(&sd->ipc_user_work_q, &wd->work_item); + if (ret < 0) + goto err_init; + } + wd->module_ref_cnt++; + return 0; + +err_init: + /* Exit if another module is already using this worker. */ + if (wd->module_ref_cnt) + return ret; + + k_thread_abort(wd->ipc_worker_tid); + user_stack_free(wd->p_worker_stack); + +err_msgq: + modules_msgq_free(wd); + +err_worker: + rfree(wd); + worker_data[sd_id] = NULL; + return ret; +} + +static void modules_user_free_worker(struct userspace_module *user) +{ + uint32_t sd_id = user->security_domain_id; + struct user_worker_data *wd = worker_data[sd_id]; + + if (sd_id >= CONFIG_SEC_DOMAIN_MAX_NUMBER || !wd) { + tr_err(&modules_user_tr, "Invalid security_domain_id"); + return; + } + + /* Module removed so decrement counter */ + wd->module_ref_cnt--; + /* Free worker resources if no more active user space modules */ + if (wd->module_ref_cnt == 0) { + k_thread_abort(wd->ipc_worker_tid); + user_stack_free(wd->p_worker_stack); + k_msgq_cleanup(wd->ipc_in_msg_q); + k_msgq_cleanup(wd->ipc_out_msg_q); + modules_msgq_free(wd); + rfree(wd); + worker_data[sd_id] = NULL; + } +} + +static int module_user_call(struct userspace_module *user, struct processing_module *mod, + uint32_t cmd, struct module_params *params) +{ + struct user_worker_data *wr_data = user->wrk_ctx; + int ret; + + params->user = user; + params->mod = mod; + params->cmd = cmd; + + /* Switch worker thread to module memory domain */ + ret = k_mem_domain_add_thread(user->comp_dom, wr_data->ipc_worker_tid); + if (ret < 0) { + comp_err(mod->dev, "failed to switch memory domain: error: %d", ret); + return ret; + } + + ret = k_msgq_put(wr_data->ipc_in_msg_q, params, K_FOREVER); + if (ret < 0) { + comp_err(mod->dev, "k_msgq_put(): error: %d", ret); + return ret; + } + + + ret = k_msgq_get(wr_data->ipc_out_msg_q, params, K_FOREVER); + if (ret < 0) + comp_err(mod->dev, "k_msgq_get(): error: %d", ret); + + return ret; +} + +static void module_user_handler(struct k_work_user *work_item) +{ + uint32_t sec_domain = CONFIG_SEC_DOMAIN_MAX_NUMBER; + const struct module_interface *ops; + + struct user_worker_data *wd = CONTAINER_OF(work_item, struct user_worker_data, work_item); + struct module_params *params = (struct module_params *)wd->ipc_params; + while(1) { + k_msgq_get(wd->ipc_in_msg_q, params, K_FOREVER); + ops = params->user->interface; + + switch(params->cmd) { + case MODULE_CMD_AGENT_START: + params->status = params->ext.agent.start_fn(params->ext.agent.entry_point, + params->ext.agent.module_id, + params->ext.agent.instance_id, + params->ext.agent.core_id, + params->ext.agent.log_handle, + ¶ms->ext.agent.mod_cfg, + ¶ms->ext.agent.iface); + break; + + case MODULE_CMD_INIT: + params->status = ops->init(params->mod); + break; + + case MODULE_CMD_PREPARE: + params->status = ops->prepare(params->mod, params->ext.proc.sources, + params->ext.proc.num_of_sources, + params->ext.proc.sinks, + params->ext.proc.num_of_sinks); + break; + + case MODULE_CMD_PROC_READY: + params->status = ops->is_ready_to_process(params->mod, + params->ext.proc.sources, + params->ext.proc.num_of_sources, + params->ext.proc.sinks, + params->ext.proc.num_of_sinks); + break; + + case MODULE_CMD_BIND: + params->status = ops->bind(params->mod, params->ext.bind_data); + break; + + case MODULE_CMD_UNBIND: + params->status = ops->unbind(params->mod, params->ext.bind_data); + break; + + case MODULE_CMD_RESET: + params->status = ops->reset(params->mod); + break; + + case MODULE_CMD_FREE: + params->status = ops->free(params->mod); + break; + + case MODULE_CMD_SET_CONF: + params->status = ops->set_configuration(params->mod, + params->ext.set_conf.config_id, + params->ext.set_conf.pos, + params->ext.set_conf.data_offset_size, + params->ext.set_conf.fragment, + params->ext.set_conf.fragment_size, + params->ext.set_conf.response, + params->ext.set_conf.response_size); + break; + + case MODULE_CMD_GET_CONF: + params->status = ops->get_configuration(params->mod, + params->ext.get_conf.config_id, + params->ext.get_conf.data_offset_size, + params->ext.get_conf.fragment, + params->ext.get_conf.fragment_size); + break; + + case MODULE_CMD_SET_PROCMOD: + params->status = ops->set_processing_mode(params->mod, + params->ext.proc_mode.mode); + break; + + case MODULE_CMD_GET_PROCMOD: + params->ext.proc_mode.mode = ops->get_processing_mode(params->mod); + break; + + case MODULE_CMD_TRIGGER: + params->status = ops->trigger(params->mod, params->ext.trigger_data); + break; + + default: + params->status = EINVAL; + break; + } + k_msgq_put(wd->ipc_out_msg_q, params, K_FOREVER); + + k_yield(); + } +} + +static int userspace_module_memory_init(struct userspace_module *user, const struct comp_driver *drv) +{ + const uintptr_t shd_addr = get_shd_heap_start(); + const size_t shd_size = get_shd_heap_size(); + uintptr_t addr_aligned; + size_t size_aligned; + + /* Add shared heap uncached and cached space to memory partitions */ + struct k_mem_partition parts[3]; + struct k_mem_partition *parts_ptr[] = { + &ipc_partition, + &parts[0], +#ifndef CONFIG_XTENSA_MMU_DOUBLE_MAP + &parts[1], +#endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ + &parts[2] + }; + k_mem_region_align(&addr_aligned, &size_aligned, shd_addr, shd_size, + CONFIG_MM_DRV_PAGE_SIZE); + parts[0].start = addr_aligned; + parts[0].size = size_aligned; + parts[0].attr = K_MEM_PARTITION_P_RW_U_RW; + +#ifndef CONFIG_XTENSA_MMU_DOUBLE_MAP + k_mem_region_align(&addr_aligned, &size_aligned, + POINTER_TO_UINT(sys_cache_cached_ptr_get(UINT_TO_POINTER(shd_addr))), + shd_size, CONFIG_MM_DRV_PAGE_SIZE); + parts[1].start = addr_aligned; + parts[1].size = size_aligned; + parts[1].attr = K_MEM_PARTITION_P_RW_U_RW; +#endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ + + /* Add module private heap to memory partitions */ + k_mem_region_align(&addr_aligned, &size_aligned, POINTER_TO_UINT(drv->drv_heap->init_mem), + DRV_HEAP_SIZE, CONFIG_MM_DRV_PAGE_SIZE); + parts[2].start = addr_aligned; + parts[2].size = size_aligned; + parts[2].attr = K_MEM_PARTITION_P_RW_U_RW; + + return k_mem_domain_init(user->comp_dom, ARRAY_SIZE(parts_ptr), parts_ptr); +} + +static int userspace_module_add_sections(struct userspace_module *user, uint32_t instance_id, + const struct sof_man_module *const mod) +{ + uint32_t mem_partition; + void* va_base; + size_t size; + int idx, ret; + + for (idx = 0; idx < ARRAY_SIZE(mod->segment); ++idx) { + if (!mod->segment[idx].flags.r.load) + continue; + + mem_partition = K_MEM_PARTITION_P_RO_U_RO; + + if (mod->segment[idx].flags.r.code) + mem_partition = K_MEM_PARTITION_P_RX_U_RX; + else if (!mod->segment[idx].flags.r.readonly) + mem_partition = K_MEM_PARTITION_P_RW_U_RW; + + va_base = UINT_TO_POINTER(mod->segment[idx].v_base_addr); + size = mod->segment[idx].flags.r.length * CONFIG_MM_DRV_PAGE_SIZE; + + ret = user_add_memory(user->comp_dom, POINTER_TO_UINT(va_base), size, mem_partition); + if (ret < 0) + return ret; + } + + lib_manager_get_instance_bss_address(instance_id, mod, &va_base, &size); + return user_add_memory(user->comp_dom, POINTER_TO_UINT(va_base), size, + K_MEM_PARTITION_P_RW_U_RW); +} + +int userspace_module_create(struct userspace_module **user_mod, const struct comp_driver *drv, + const struct sof_man_module *manifest, system_agent_start_fn agent, + uintptr_t entry_point, uint32_t module_id, uint32_t instance_id, + uint32_t core_id, uint32_t log_handle, byte_array_t *mod_cfg, + void **adapter) +{ + struct userspace_module *user; + struct k_mem_domain *domain; + int ret; + + tr_dbg(&modules_user_tr, "userspace create"); + + /* Allocate memory domain struct */ + user = sys_heap_alloc(drv->drv_heap, sizeof(struct userspace_module)); + if (!user) + return -ENOMEM; + + domain = rzalloc(SOF_MEM_ZONE_SYS_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*domain)); + if (!domain) { + ret = -ENOMEM; + goto error; + } + user->comp_dom = domain; + + ret = userspace_module_memory_init(user, drv); + if (ret) + goto error_dom; + + ret = userspace_module_add_sections(user, instance_id, manifest); + if (ret) + goto error_dom; + + ret = modules_user_init_worker(user, module_user_handler); + if (ret) + goto error_dom; + + struct user_worker_data *wr_data = user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + params->ext.agent.start_fn = agent; + params->ext.agent.entry_point = entry_point; + params->ext.agent.module_id = module_id; + params->ext.agent.instance_id = instance_id; + params->ext.agent.core_id = core_id; + params->ext.agent.log_handle = log_handle; + params->ext.agent.mod_cfg = *mod_cfg; + + /* Add cfg buffer to memory domain */ + ret = user_add_memory(domain, POINTER_TO_UINT(mod_cfg->data), mod_cfg->size << 2, + K_MEM_PARTITION_P_RW_U_RW); + if (ret) + goto error_worker; + + ret = module_user_call(user, NULL, MODULE_CMD_AGENT_START, params); + if (ret) + goto error_worker; + + /* Remove cfg buffer from memory domain (Is this needed?) */ + ret = user_remove_memory(domain, POINTER_TO_UINT(mod_cfg->data), + mod_cfg->size << 2); + if (ret) + goto error_worker; + + *adapter = params->ext.agent.iface; + *user_mod = user; + + /* TODO: Must be after assign of adapter! */ + user->interface = drv->adapter_ops; + ((struct comp_driver *)drv)->adapter_ops = &userspace_module_adapter_interface; + + return params->status; + +error_worker: + modules_user_free_worker(user); +error_dom: + rfree(domain); +error: + sys_heap_free(drv->drv_heap, user); + return ret; +} + +void userspace_module_destroy(const struct comp_driver *drv, struct userspace_module *mod) +{ + tr_dbg(&modules_user_tr, "userspace free"); + modules_user_free_worker(mod); + sys_heap_free(drv->drv_heap, mod); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module init() operation and return its result. + * Module init() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @param module_entry_point - module entry point address from + * its manifest + * @param module_id - module id + * @param instance_id - instance id + * @param log_handle - log handler for module + * @param mod_cfg - module configuration + * @return 0 for success, error otherwise. + */ +static int userspace_module_init(struct processing_module *mod) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_data *md = &mod->priv; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + struct k_mem_domain *domain = mod->dev_user->comp_dom; + int ret; + + comp_dbg(mod->dev, "start"); + + /* Add cfg buffer to memory domain */ + ret = user_add_memory(domain, POINTER_TO_UINT(md->cfg.init_data), + sizeof(struct module_config), K_MEM_PARTITION_P_RW_U_RW); + if (ret < 0) + return ret; + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_INIT, params); + if (ret < 0) + return ret; + + /* Remove cfg buffer from memory domain (Is this needed?) */ + ret = user_remove_memory(domain, POINTER_TO_UINT(md->cfg.init_data), + sizeof(struct module_config)); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module prepare() operation and return its result. + * Module prepare() code is performed in user workqueue. + * + * @return 0 for success, error otherwise. + */ +static int userspace_module_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->prepare) + return 0; + + params->ext.proc.sources = sources; + params->ext.proc.num_of_sources = num_of_sources; + params->ext.proc.sinks = sinks; + params->ext.proc.num_of_sinks = num_of_sinks; + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_PREPARE, params); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module init() operation and return its result. + * Module init() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @param module_entry_point - module entry point address from + * its manifest + * @param module_id - module id + * @param instance_id - instance id + * @param log_handle - log handler for module + * @param mod_cfg - module configuration + * @return 0 for success, error otherwise. + */ +static int userspace_module_process(struct processing_module *mod, struct sof_source **sources, + int num_of_sources, struct sof_sink **sinks, int num_of_sinks) +{ + return mod->dev_user->interface->process(mod, sources, num_of_sources, sinks, num_of_sinks); +} + +/** + * Copy parameters to user worker accessible space. + * Queue module reset() operation and return its result. + * Module reset() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_module_reset(struct processing_module *mod) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->reset) + return 0; + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_RESET, params); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module free() operation and return its result. + * Module free() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_module_free(struct processing_module *mod) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + params->status = 0; + + if (mod->dev_user->interface->free) { + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_FREE, params); + if (ret < 0) + return ret; + } + + /* Destroy workqueue if this was last active userspace module */ + userspace_module_destroy(mod->dev->drv, mod->dev_user); + mod->dev_user = NULL; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module set_configuration() operation and return its result. + * Module set_configuration() code is performed in user workqueue. + * + * @param[in] mod - struct processing_module pointer + * @param[in] config_id - Configuration ID + * @param[in] pos - position of the fragment in the large message + * @param[in] data_offset_size: size of the whole configuration if it is the first fragment or the + * only fragment. Otherwise, it is the offset of the fragment in the whole + * configuration. + * @param[in] fragment: configuration fragment buffer + * @param[in] fragment_size: size of @fragment + * @params[in] response: optional response buffer to fill + * @params[in] response_size: size of @response + * + * @return 0 for success, error otherwise. + */ +static int userspace_module_set_configuration(struct processing_module *mod, uint32_t config_id, + enum module_cfg_fragment_position pos, + uint32_t data_offset_size, const uint8_t *fragment, + size_t fragment_size, uint8_t *response, + size_t response_size) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + struct k_mem_domain *domain = mod->dev_user->comp_dom; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->set_configuration) + return 0; + + params->ext.set_conf.config_id = config_id; + params->ext.set_conf.pos = pos; + params->ext.set_conf.data_offset_size = data_offset_size; + params->ext.set_conf.fragment = fragment; + params->ext.set_conf.fragment_size = fragment_size; + params->ext.set_conf.response = response; + params->ext.set_conf.response_size = response_size; + + /* Give read access to the fragment buffer (It should be 4kB)*/ + ret = user_add_memory(domain, POINTER_TO_UINT(fragment), fragment_size, + K_MEM_PARTITION_P_RO_U_RO); + if (ret < 0) { + comp_err(mod->dev, "add fragment to domain error: %d", ret); + return ret; + } + + /* Give write access to the response buffer (It should be 4kB)*/ + ret = user_add_memory(domain, POINTER_TO_UINT(response), response_size, + K_MEM_PARTITION_P_RW_U_RW); + if (ret < 0) { + comp_err(mod->dev, "add response to domain error: %d", ret); + goto err_resp; + } + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_SET_CONF, params); + if (!ret) + ret = params->status; + + /* Remove access to buffers */ + user_remove_memory(domain, POINTER_TO_UINT(response), response_size); +err_resp: + user_remove_memory(domain, POINTER_TO_UINT(fragment), fragment_size); + + /* Return status from module code operation. */ + return ret; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module get_configuration() operation and return its result. + * Module get_configuration() code is performed in user workqueue. + * + * @param[in] mod - struct processing_module pointer + * @param[in] config_id - Configuration ID + * @param[in] data_offset_size: size of the whole configuration if it is the first fragment or the + * only fragment. Otherwise, it is the offset of the fragment in the whole + * configuration. + * @param[in] fragment: configuration fragment buffer + * @param[in] fragment_size: size of @fragment + * + * @return 0 for success, error otherwise. + */ +static int userspace_module_get_configuration(struct processing_module *mod, uint32_t config_id, + uint32_t *data_offset_size, uint8_t *fragment, + size_t fragment_size) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + struct k_mem_domain *domain = mod->dev_user->comp_dom; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->get_configuration) + return -EIO; + + params->ext.get_conf.config_id = config_id; + params->ext.get_conf.data_offset_size = data_offset_size; + params->ext.get_conf.fragment = fragment; + params->ext.get_conf.fragment_size = fragment_size; + + /* Give write access to the fragment buffer (It should be 4kB)*/ + ret = user_add_memory(domain, POINTER_TO_UINT(fragment), fragment_size, + K_MEM_PARTITION_P_RW_U_RW); + if (ret < 0) { + comp_err(mod->dev, "add fragment to domain error: %d", ret); + return ret; + } + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_GET_CONF, params); + + /* Remove access to the buffer */ + user_remove_memory(domain, POINTER_TO_UINT(fragment), fragment_size); + + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module set_processing_mode() operation and return its result. + * Module set_processing_mode() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @param mode - processing mode to be set. + * @return 0 for success, error otherwise. + */ +static int userspace_module_set_processing_mode(struct processing_module *mod, + enum module_processing_mode mode) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->set_processing_mode) + return 0; + + params->ext.proc_mode.mode = mode; + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_SET_PROCMOD, params); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module get_processing_mode() operation and return its result. + * Module get_processing_mode() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return processing mode. + */ +static enum module_processing_mode userspace_module_get_processing_mode(struct processing_module *mod) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->get_processing_mode) + return -EIO; + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_GET_PROCMOD, params); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->ext.proc_mode.mode; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module is_ready_to_process() operation and return its result. + * Module is_ready_to_process() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return true if the module is ready to process + */ +static bool userspace_module_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, + int num_of_sources, + struct sof_sink **sinks, + int num_of_sinks) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->is_ready_to_process) + return generic_module_is_ready_to_process(mod, sources, num_of_sources, sinks, + num_of_sinks); + + params->ext.proc.sources = sources; + params->ext.proc.num_of_sources = num_of_sources; + params->ext.proc.sinks = sinks; + params->ext.proc.num_of_sinks = num_of_sinks; + + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_PROC_READY, params); + if (ret < 0) + return generic_module_is_ready_to_process(mod, sources, num_of_sources, sinks, + num_of_sinks); + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module bind() operation and return its result. + * Module bind() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_module_bind(struct processing_module *mod, void *data) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->bind) + return 0; + + params->ext.bind_data = data; + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_BIND, params); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module unbind() operation and return its result. + * Module unbind() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_module_unbind(struct processing_module *mod, void *data) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret; + + comp_dbg(mod->dev, "start"); + + if (!mod->dev_user->interface->unbind) + return 0; + + params->ext.bind_data = data; + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_UNBIND, params); + if (ret < 0) + return ret; + + /* Return status from module code operation. */ + return params->status; +} + +/** + * Copy parameters to user worker accessible space. + * Queue module trigger() operation and return its result. + * Module trigger() code is performed in user workqueue. + * + * @param mod - pointer to processing module structure. + * @return 0 for success, error otherwise. + */ +static int userspace_module_trigger(struct processing_module *mod, int cmd) +{ + struct user_worker_data *wr_data = mod->dev_user->wrk_ctx; + struct module_params *params = (struct module_params *)wr_data->ipc_params; + int ret = 0; + + comp_dbg(mod->dev, "start"); + + if (mod->dev_user->interface->trigger) { + params->ext.trigger_data = cmd; + ret = module_user_call(mod->dev_user, mod, MODULE_CMD_TRIGGER, params); + if (ret < 0) + return ret; + ret = params->status; + } + + if (!ret) + ret = module_adapter_set_state(mod, mod->dev, cmd); + + /* Return status from module code operation. */ + return ret; +} + +/* Userspace Module Adapter API */ +APP_USER_DATA static const struct module_interface userspace_module_adapter_interface = { + .init = userspace_module_init, + .is_ready_to_process = userspace_module_is_ready_to_process, + .prepare = userspace_module_prepare, + .process = userspace_module_process, + .set_configuration = userspace_module_set_configuration, + .get_configuration = userspace_module_get_configuration, + .set_processing_mode = userspace_module_set_processing_mode, + .get_processing_mode = userspace_module_get_processing_mode, + .reset = userspace_module_reset, + .free = userspace_module_free, + .bind = userspace_module_bind, + .unbind = userspace_module_unbind, + .trigger = userspace_module_trigger, +}; diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 65ffd97ac18b..05f7bc705599 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -41,7 +41,7 @@ LOG_MODULE_REGISTER(module_adapter, CONFIG_SOF_LOG_LEVEL); struct comp_dev *module_adapter_new(const struct comp_driver *drv, const struct comp_ipc_config *config, const void *spec) { - return module_adapter_new_ext(drv, config, spec, NULL); + return module_adapter_new_ext(drv, config, spec, NULL, NULL); } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, @@ -108,7 +108,7 @@ static void module_adapter_mem_free(struct processing_module *mod) */ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, const struct comp_ipc_config *config, const void *spec, - void *mod_priv) + void *mod_priv, struct userspace_context *user_ctx) { int ret; struct module_config *dst; @@ -127,14 +127,16 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, if (!mod) return NULL; - dst = &mod->priv.cfg; module_set_private_data(mod, mod_priv); + list_init(&mod->raw_data_buffers_list); +#if CONFIG_USERSPACE + mod->user_ctx = user_ctx; +#endif /* CONFIG_USERSPACE */ struct comp_dev *dev = mod->dev; - list_init(&mod->raw_data_buffers_list); - + dst = &mod->priv.cfg; ret = module_adapter_init_data(dev, dst, config, spec); if (ret) { comp_err(dev, "%d: module init data failed", diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 2d0e9faba610..02639f823419 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -1214,7 +1214,7 @@ bool comp_update_performance_data(struct comp_dev *dev, uint32_t cycles_used); static inline int user_get_buffer_memory_region(const struct comp_driver *drv) { -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP return drv->user_heap ? SOF_MEM_FLAG_USER_SHARED_BUFFER : SOF_MEM_FLAG_USER; #else return SOF_MEM_FLAG_USER; diff --git a/src/include/sof/audio/module_adapter/iadk/system_agent.h b/src/include/sof/audio/module_adapter/iadk/system_agent.h index a0470167c796..e7aab98c748c 100644 --- a/src/include/sof/audio/module_adapter/iadk/system_agent.h +++ b/src/include/sof/audio/module_adapter/iadk/system_agent.h @@ -107,8 +107,9 @@ extern "C" { * method (the variant with 7 parameters) via a parameter that initially contained the address to * the agent system. The system_agent_start function returns it in the variable adapter. */ -int system_agent_start(uintptr_t entry_point, uint32_t module_id, uint32_t instance_id, - uint32_t core_id, uint32_t log_handle, void *mod_cfg, const void **adapter); +struct system_agent_params; + +int system_agent_start(const struct system_agent_params *params, const void **adapter); #ifdef __cplusplus } #endif diff --git a/src/include/sof/audio/module_adapter/library/native_system_agent.h b/src/include/sof/audio/module_adapter/library/native_system_agent.h index 657bc6ac643d..70c0f6b1aad1 100644 --- a/src/include/sof/audio/module_adapter/library/native_system_agent.h +++ b/src/include/sof/audio/module_adapter/library/native_system_agent.h @@ -11,9 +11,17 @@ #include #include -typedef int (*system_agent_start_fn)(uintptr_t entry_point, uint32_t module_id, - uint32_t instance_id, uint32_t core_id, uint32_t log_handle, - void *mod_cfg, const void **adapter); +struct system_agent_params { + uintptr_t entry_point; /* The module entry point function address. */ + uint32_t module_id; /* The identifier for the module. */ + uint32_t instance_id; /* The instance identifier of the module. */ + uint32_t core_id; /* Core on which the module will run. */ + uint32_t log_handle; /* The handle for logging purposes. */ + void *mod_cfg; /* Pointer to the module configuration data. */ +}; + +typedef int (*system_agent_start_fn)(const struct system_agent_params *params, + const void **adapter); struct native_system_agent { struct system_service system_service; @@ -29,18 +37,12 @@ struct native_system_agent { * * This function initializes and starts the native system agent with the provided parameters. * - * @param[in] entry_point - The module entry point function address. - * @param[in] module_id - The identifier for the module. - * @param[in] instance_id - The instance identifier of the module. - * @param[in] core_id - Core on which the module will run. - * @param[in] log_handle - The handle for logging purposes. - * @param[in] mod_cfg - Pointer to the module configuration data. + * @param[in] params - Pointer to the system agent parameter structure * @param[out] iface - Pointer to the module interface. * * @return Returns 0 on success or an error code on failure. */ -int native_system_agent_start(uintptr_t entry_point, uint32_t module_id, uint32_t instance_id, - uint32_t core_id, uint32_t log_handle, void *mod_cfg, +int native_system_agent_start(const struct system_agent_params *params, const void **iface); #endif /* __NATIVE_SYSTEM_AGENT_H__ */ diff --git a/src/include/sof/audio/module_adapter/library/userspace_proxy.h b/src/include/sof/audio/module_adapter/library/userspace_proxy.h index 52381ac7c175..150812183a90 100644 --- a/src/include/sof/audio/module_adapter/library/userspace_proxy.h +++ b/src/include/sof/audio/module_adapter/library/userspace_proxy.h @@ -14,12 +14,52 @@ #include #include +#include + +#include +#include + +struct module_interface; +struct comp_driver; +struct sof_man_module; +struct system_agent_params; /* Processing module structure fields needed for user mode */ struct userspace_context { struct k_mem_domain *comp_dom; /* Module specific memory domain */ + const struct module_interface *interface; /* Userspace module interface */ }; - #endif /* CONFIG_USERSPACE */ +#if CONFIG_SOF_USERSPACE_PROXY +/** + * Creates userspace module proxy + * + * @param user_ctx - pointer to pointer of userspace module context + * @param drv - pointer to component driver + * @param manifest - pointer to module manifest + * @param start_fn - pointer to system agent start function + * @param agent_params - pointer to system_agent_params + * @param agent_interface - pointer to variable to store module interface created by agent + * @param ops - Pointer to a variable that will hold the address of the module interface + * structure. The function stores a pointer to its own userspace proxy + * interface structure in this variable. + * + * @return 0 for success, error otherwise. + */ +int userspace_proxy_create(struct userspace_context **user_ctx, const struct comp_driver *drv, + const struct sof_man_module *manifest, system_agent_start_fn start_fn, + const struct system_agent_params *agent_params, + const void **agent_interface, const struct module_interface **ops); + +/** + * Destroy userspace module proxy + * + * @param drv - pointer to component driver + * @param user_ctx - pointer to userspace module context + */ +void userspace_proxy_destroy(const struct comp_driver *drv, struct userspace_context *user_ctx); + +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + #endif /* __SOF_AUDIO_USERSPACE_PROXY_H__ */ diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index c1e730893517..484c0d6856f7 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -194,7 +194,7 @@ void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, /** * Allocates aligned memory block for module. * @param mod Pointer to the module this memory block is allocated for. - * @param bytes Size in bytes. + * @param size Size in bytes. * @param alignment Alignment in bytes. * @return Pointer to the allocated memory or NULL if failed. * @@ -300,9 +300,10 @@ int module_unbind(struct processing_module *mod, struct bind_info *unbind_data); struct comp_dev *module_adapter_new(const struct comp_driver *drv, const struct comp_ipc_config *config, const void *spec); +struct userspace_context; struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, const struct comp_ipc_config *config, const void *spec, - void *mod_priv); + void *mod_priv, struct userspace_context *user_ctx); int module_adapter_prepare(struct comp_dev *dev); int module_adapter_params(struct comp_dev *dev, struct sof_ipc_stream_params *params); int module_adapter_copy(struct comp_dev *dev); diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 9e7de75a88ee..8bdc8545c808 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -3,7 +3,8 @@ // Copyright(c) 2022 Intel Corporation. All rights reserved. // // Author: Jaroslaw Stelter -// Pawel Dobrowolski +// Pawel Dobrowolski +// Adrian Warecki /* * Dynamic module loading functions. @@ -18,11 +19,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -492,18 +495,21 @@ const struct sof_man_module *lib_manager_get_module_manifest(const uint32_t modu * \param[in] module_entry_point - Entry point address of the module. * \param[in] agent - Function pointer to the system agent start function. * \param[out] agent_interface - Pointer to store the module interface returned by the system agent. + * \param[out] userspace - Pointer to store the userspace module proxy context. + * \param[out] ops - Pointer to store pointer to the module interface structure. * * \return Error code returned by the system agent, 0 on success. */ static int lib_manager_start_agent(const struct comp_driver *drv, const uint32_t component_id, + const struct sof_man_module *mod_manifest, const struct ipc_config_process *args, const uintptr_t module_entry_point, const system_agent_start_fn agent, - const void **agent_interface) + const void **agent_interface, + struct userspace_context **userspace, + const struct module_interface **ops) { - const uint32_t module_id = IPC4_MOD_ID(component_id); - const uint32_t instance_id = IPC4_INST_ID(component_id); - const uint32_t log_handle = (uint32_t)drv->tctx; + struct system_agent_params agent_params; byte_array_t mod_cfg; int ret; @@ -511,8 +517,25 @@ static int lib_manager_start_agent(const struct comp_driver *drv, const uint32_t /* Intel modules expects DW size here */ mod_cfg.size = args->size >> 2; - ret = agent(module_entry_point, module_id, instance_id, 0, log_handle, &mod_cfg, - agent_interface); + agent_params.entry_point = module_entry_point; + agent_params.module_id = IPC4_MOD_ID(component_id); + agent_params.instance_id = IPC4_INST_ID(component_id); + agent_params.core_id = 0; + agent_params.log_handle = (uint32_t)drv->tctx; + agent_params.mod_cfg = &mod_cfg; + +#if CONFIG_SOF_USERSPACE_PROXY + /* If drv->user_heap is allocated, it means the module is userspace. */ + if (drv->user_heap) { + ret = userspace_proxy_create(userspace, drv, mod_manifest, agent, &agent_params, + agent_interface, ops); + if (ret) + tr_err(&lib_manager_tr, "userspace_proxy_create failed! %d", ret); + return ret; + } +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + + ret = agent(&agent_params, agent_interface); if (ret) tr_err(&lib_manager_tr, "System agent start failed %d!", ret); @@ -580,7 +603,8 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); - const struct module_interface *ops = NULL; + struct userspace_context *userspace = NULL; + const struct module_interface *ops; const struct sof_man_module *mod; system_agent_start_fn agent; void *adapter_priv = NULL; @@ -588,6 +612,13 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, struct comp_dev *dev; int ret; +#ifdef CONFIG_SOF_USERSPACE_PROXY + if (drv->user_heap && config->proc_domain != COMP_PROCESSING_DOMAIN_DP) { + tr_err(&lib_manager_tr, "Userspace supports only DP modules."); + return NULL; + } +#endif + tr_dbg(&lib_manager_tr, "start"); if (!desc) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", @@ -619,35 +650,39 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, agent = &native_system_agent_start; agent_iface = (const void **)&ops; break; +#if CONFIG_INTEL_MODULES case MOD_TYPE_IADK: agent = &system_agent_start; - /* processing_module_adapter_interface is already assigned to drv->adapter_ops by - * the lib_manager_prepare_module_adapter function. - */ + ops = &processing_module_adapter_interface; agent_iface = (const void **)&adapter_priv; break; +#endif case MOD_TYPE_INVALID: goto err; } /* At this point module resources are allocated and it is moved to L2 memory. */ if (agent) { - ret = lib_manager_start_agent(drv, config->id, args, module_entry_point, agent, - agent_iface); + ret = lib_manager_start_agent(drv, config->id, mod, args, module_entry_point, agent, + agent_iface, &userspace, &ops); if (ret) goto err; } - if (ops && comp_set_adapter_ops(drv, ops) < 0) + if (comp_set_adapter_ops(drv, ops) < 0) goto err; - dev = module_adapter_new_ext(drv, config, spec, adapter_priv); + dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace); if (!dev) goto err; return dev; err: +#if CONFIG_SOF_USERSPACE_PROXY + if (userspace) + userspace_proxy_destroy(drv, userspace); +#endif /* CONFIG_SOF_USERSPACE_PROXY */ lib_manager_free_module(config->id); return NULL; } @@ -694,18 +729,16 @@ static void lib_manager_prepare_module_adapter(struct comp_driver *drv, const st drv->ops.dai_ts_start = module_adapter_ts_start_op; drv->ops.dai_ts_stop = module_adapter_ts_stop_op; drv->ops.dai_ts_get = module_adapter_ts_get_op; -#if CONFIG_INTEL_MODULES - drv->adapter_ops = &processing_module_adapter_interface; -#endif } int lib_manager_register_module(const uint32_t component_id) { - struct comp_driver_info *new_drv_info; - struct comp_driver *drv = NULL; const struct sof_man_module *mod = lib_manager_get_module_manifest(component_id); const struct sof_uuid *uid = (struct sof_uuid *)&mod->uuid; - int ret; + struct comp_driver_info *new_drv_info; + struct sys_heap *drv_heap = NULL; + struct comp_driver *drv = NULL; + int ret = -ENOMEM; if (!mod) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", @@ -722,13 +755,23 @@ int lib_manager_register_module(const uint32_t component_id) return -ENOMEM; } - drv = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(struct comp_driver)); +#if CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP + if (mod->type.user_mode) { + drv_heap = module_driver_heap_init(); + if (!drv_heap) { + tr_err(&lib_manager_tr, "failed to allocate driver heap!"); + goto cleanup; + } + } +#endif /* CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP */ + + drv = module_driver_heap_rmalloc(drv_heap, SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(struct comp_driver)); if (!drv) { tr_err(&lib_manager_tr, "failed to allocate comp_driver"); - ret = -ENOMEM; goto cleanup; } + drv->user_heap = drv_heap; lib_manager_prepare_module_adapter(drv, uid); @@ -741,8 +784,9 @@ int lib_manager_register_module(const uint32_t component_id) cleanup: if (ret < 0) { - rfree(drv); rfree(new_drv_info); + module_driver_heap_free(drv_heap, drv); + module_driver_heap_remove(drv_heap); } return ret; diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 991d9a30eb49..28012254ef35 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -25,6 +25,9 @@ #include +extern volatile uint32_t debug_val[4]; +extern volatile uint32_t *debug_ptr[16]; + LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(dp_sched); @@ -397,73 +400,112 @@ static int scheduler_dp_task_free(void *data, struct task *task) /* all other memory has been allocated as a single malloc, will be freed later by caller */ return ret; } - +__attribute__((optimize("-O0"))) /* Thread function called in component context, on target core */ static void dp_thread_fn(void *p1, void *p2, void *p3) { struct task *task = p1; (void)p2; (void)p3; - struct task_dp_pdata *task_pdata = task->priv_data; unsigned int lock_key; enum task_state state; bool task_stop; - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); + struct scheduler_dp_data *dp_sch = NULL; + volatile int dbg = __LINE__; + struct task_dp_pdata *task_pdata = NULL; + + //while (dbg); + + dbg = __LINE__; + task_pdata = task->priv_data; + dbg = __LINE__; + + if (!(task->flags & K_USER)) + dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); + dbg = __LINE__; do { /* * the thread is started immediately after creation, it will stop on semaphore * Semaphore will be released once the task is ready to process */ + dbg = __LINE__; k_sem_take(task_pdata->sem, K_FOREVER); + dbg = __LINE__; - if (task->state == SOF_TASK_STATE_RUNNING) + if (task->state == SOF_TASK_STATE_RUNNING) { + dbg = __LINE__; state = task_run(task); - else + } else { + dbg = __LINE__; state = task->state; /* to avoid undefined variable warning */ + } + dbg = __LINE__; lock_key = scheduler_dp_lock(task->core); + dbg = __LINE__; /* * check if task is still running, may have been canceled by external call * if not, set the state returned by run procedure */ if (task->state == SOF_TASK_STATE_RUNNING) { + dbg = __LINE__; task->state = state; switch (state) { case SOF_TASK_STATE_RESCHEDULE: + dbg = __LINE__; /* mark to reschedule, schedule time is already calculated */ task->state = SOF_TASK_STATE_QUEUED; + dbg = __LINE__; break; case SOF_TASK_STATE_CANCEL: + dbg = __LINE__; case SOF_TASK_STATE_COMPLETED: /* remove from scheduling */ + dbg = __LINE__; list_item_del(&task->list); + dbg = __LINE__; break; default: + dbg = __LINE__; /* illegal state, serious defect, won't happen */ k_panic(); } } /* if true exit the while loop, terminate the thread */ + dbg = __LINE__; task_stop = task->state == SOF_TASK_STATE_COMPLETED || task->state == SOF_TASK_STATE_CANCEL; + dbg = __LINE__; /* recalculate all DP tasks readiness and deadlines * TODO: it should be for all tasks, for all cores * currently its limited to current core only */ - scheduler_dp_recalculate(dp_sch, false); + dbg = __LINE__; + if (dp_sch) { + dbg = __LINE__; + scheduler_dp_recalculate(dp_sch, false); + } + dbg = __LINE__; scheduler_dp_unlock(lock_key); + dbg = __LINE__; + } while (!task_stop); + dbg = __LINE__; /* call task_complete */ - if (task->state == SOF_TASK_STATE_COMPLETED) + if (task->state == SOF_TASK_STATE_COMPLETED) { + dbg = __LINE__; task_complete(task); -} + } + dbg = __LINE__; +} +__attribute__((optimize("-O0"))) static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, uint64_t period) { @@ -472,6 +514,9 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta unsigned int lock_key; int ret; + volatile int i = 0; + while (i); + lock_key = scheduler_dp_lock(cpu_get_id()); if (task->state != SOF_TASK_STATE_INIT && @@ -481,6 +526,13 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return -EINVAL; } + debug_val[0] = sizeof(*task); + debug_val[1] = (uint32_t)task; + debug_val[2] = (uint32_t)task->priv_data; + debug_val[3] = (uint32_t)pdata->p_stack; + debug_val[4] = (uint32_t)pdata->p_stack + pdata->stack_size; + //debug_ptr[0] = &task->priv_data; + /* create a zephyr thread for the task */ pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)pdata->p_stack, pdata->stack_size, dp_thread_fn, task, NULL, NULL, @@ -568,7 +620,7 @@ int scheduler_dp_init(void) return 0; } - +__attribute__((optimize("-O0"))) int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, const struct task_ops *ops, diff --git a/uuid-registry.txt b/uuid-registry.txt index 99a1a34df3db..84ded352b44f 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -170,6 +170,7 @@ c51dc642-a2e1-48df-a490e2748cb6363e tflmcly 04e3f894-2c5c-4f2e-8dc1694eeaab53fa tone e93326d8-0d14-4bf0-bcb9e063d3d80136 twb_sched 42f8060c-832f-4dbf-b24751e961997b34 up_down_mixer +6f6b6f4b-6f73-7466-20e1e62b9779f003 userspace_proxy b77e677e-5ff4-4188-af14fba8bdbf8682 volume 8a171323-94a3-4e1d-afe9fe5dbaa4c393 volume4 1028070e-04e8-46ab-8d8110a0116ce738 wait diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 1314454a0466..854eb211e904 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -67,6 +67,14 @@ config SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE help This config defines size of virtual heap region shared between all cores +config SOF_USERSPACE_USE_DRIVER_HEAP + bool "Use driver heap for SOF userspace modules" + depends on USERSPACE + help + When selected, multiple instances of the same userspace module will share + a private heap created for that module's driver. Otherwise, each module + instance will have its own independent private heap. + config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE hex "Size of the private heap created for each userspace module" default 0x1000 @@ -76,6 +84,14 @@ config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE module has its own independent heap to which only it has access. This heap is shared between instances of the same module. +config SOF_USERSPACE_PROXY + bool "Use userspace proxy to support userspace modules" + select SOF_USERSPACE_USE_DRIVER_HEAP + help + When set, userspace modules are launched inside a container created by userspace proxy. + It is responsible for forwarding module function calls coming from sof running in + kernelspace to the module code executed with user privileges. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" help diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h index 254c8a2f96f1..7ceaa6411137 100644 --- a/zephyr/include/rtos/userspace_helper.h +++ b/zephyr/include/rtos/userspace_helper.h @@ -19,8 +19,8 @@ #include -#define DRV_HEAP_SIZE ALIGN_UP(CONFIG_SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE, \ - CONFIG_MM_DRV_PAGE_SIZE) +#define USER_MOD_HEAP_SIZE ALIGN_UP(CONFIG_SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE, \ + CONFIG_MM_DRV_PAGE_SIZE) #define APP_TASK_BSS K_APP_BMEM(common_partition) #define APP_TASK_DATA K_APP_DMEM(common_partition) @@ -40,6 +40,38 @@ struct userspace_context; */ struct sys_heap *module_driver_heap_init(void); +/** + * Add memory region to non-privileged module memory domain. + * @param domain - pointer to the modules memory domain. + * @param addr - pointer to memory region start + * @param size - size of the memory region + * @param attr - memory region access attributes + * + * @return 0 for success, error otherwise. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * Function adds page aligned region to the memory domain. + * Caller should take care to not expose other data than these + * intended to be shared with the module. + */ +int user_add_memory(struct k_mem_domain *domain, uintptr_t addr, size_t size, uint32_t attr); + +/** + * Remove memory region from non-privileged module memory domain. + * @param domain - pointer to the modules memory domain. + * @param addr - pointer to memory region start + * @param size - size of the memory region + * + * @return 0 for success, error otherwise. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * Function removes previously added page aligned region + * from the memory domain. + */ +int user_remove_memory(struct k_mem_domain *domain, uintptr_t addr, size_t size); + /** * Add DP scheduler created thread to module memory domain. * @param thread_id - id of thread to be added to memory domain. @@ -133,4 +165,13 @@ void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem); */ void module_driver_heap_remove(struct sys_heap *mod_drv_heap); +void dump_memory_domain(struct k_mem_domain *domain); +void dump_page_tables(uint32_t *ptables, void *test, bool kernel); +void dump_page_table(uint32_t *ptables, void *test); + +static inline void dump_domain_entry(struct k_mem_domain *domain, void *test) +{ + dump_page_tables(domain->arch.ptables, test, false); +} + #endif /* __ZEPHYR_LIB_USERSPACE_HELPER_H__ */ diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index b26370b12d25..7ebc851e85f0 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -24,6 +24,7 @@ /* Zephyr includes */ #include #include +#include <../../../../../zephyr/zephyr/arch/xtensa/include/xtensa_mmu_priv.h> #if CONFIG_USERSPACE @@ -36,16 +37,16 @@ struct sys_heap *module_driver_heap_init(void) if (!mod_drv_heap) return NULL; - void *mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, DRV_HEAP_SIZE, + void *mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, USER_MOD_HEAP_SIZE, CONFIG_MM_DRV_PAGE_SIZE); if (!mem) { rfree(mod_drv_heap); return NULL; } - sys_heap_init(mod_drv_heap, mem, DRV_HEAP_SIZE); + sys_heap_init(mod_drv_heap, mem, USER_MOD_HEAP_SIZE); mod_drv_heap->init_mem = mem; - mod_drv_heap->init_bytes = DRV_HEAP_SIZE; + mod_drv_heap->init_bytes = USER_MOD_HEAP_SIZE; return mod_drv_heap; } @@ -156,6 +157,147 @@ int user_memory_init_shared(k_tid_t thread_id, struct processing_module *mod) return k_mem_domain_add_thread(comp_dom, thread_id); } +int user_add_memory(struct k_mem_domain *domain, uintptr_t addr, size_t size, uint32_t attr) +{ + struct k_mem_partition part; + int ret; + + k_mem_region_align(&part.start, &part.size, addr, size, HOST_PAGE_SIZE); + /* Define parameters for partition */ + part.attr = attr; + ret = k_mem_domain_add_partition(domain, &part); + /* -EINVAL means that given page is already in the domain */ + /* Not an error case for us. */ + if (ret == -EINVAL) + return 0; + + return ret; +} + +int user_remove_memory(struct k_mem_domain *domain, uintptr_t addr, size_t size) +{ + struct k_mem_partition part; + int ret; + + /* Define parameters for user_partition */ + k_mem_region_align(&part.start, &part.size, addr, size, HOST_PAGE_SIZE); + ret = k_mem_domain_remove_partition(domain, &part); + /* -ENOENT means that given partition is already removed */ + /* Not an error case for us. */ + if (ret == -ENOENT) + return 0; + + return ret; +} + +extern struct tr_ctx userspace_proxy_tr; + +LOG_MODULE_DECLARE(userspace_proxy, CONFIG_SOF_LOG_LEVEL); + +static void dump_pte_attr(char *attr_s, uint32_t attr) +{ + attr_s[0] = attr & XTENSA_MMU_CACHED_WT ? 'T' : '-'; + attr_s[1] = attr & XTENSA_MMU_CACHED_WB ? 'B' : '-'; + attr_s[2] = attr & XTENSA_MMU_PERM_W ? 'W' : '-'; + attr_s[3] = attr & XTENSA_MMU_PERM_X ? 'X' : '-'; + attr_s[4] = '\0'; +} + +static uint32_t *dump_pte(uint32_t pte) +{ + uint32_t ppn = pte & XTENSA_MMU_PTE_PPN_MASK; + uint32_t ring = XTENSA_MMU_PTE_RING_GET(pte); + uint32_t sw = XTENSA_MMU_PTE_SW_GET(pte); + uint32_t sw_ring = XTENSA_MMU_PTE_SW_RING_GET(sw); + uint32_t sw_attr = XTENSA_MMU_PTE_SW_ATTR_GET(sw); + uint32_t attr = XTENSA_MMU_PTE_ATTR_GET(pte); + + char attr_s[5]; + dump_pte_attr(attr_s, attr); + + char sw_attr_s[5]; + dump_pte_attr(sw_attr_s, sw_attr); + + tr_err(&userspace_proxy_tr, "PPN %#x, sw %#x (ring: %u, %s), ring %u %s", + ppn, sw, sw_ring, sw_attr_s, ring, attr_s); + + if ((attr & XTENSA_MMU_PTE_ATTR_ILLEGAL) == XTENSA_MMU_PTE_ATTR_ILLEGAL) { + tr_err(&userspace_proxy_tr, "ILLEGAL PTE"); + return NULL; + } + + return (uint32_t *)ppn; +} + +void dump_page_table(uint32_t *ptables, void *test) +{ + const uint32_t l1_index = XTENSA_MMU_L1_POS(POINTER_TO_UINT(test)); + const uint32_t l2_index = XTENSA_MMU_L2_POS(POINTER_TO_UINT(test)); + const uint32_t test_aligned = POINTER_TO_UINT(test) & ~(CONFIG_MMU_PAGE_SIZE - 1); + + tr_err(&userspace_proxy_tr, "test %p, ptables = %p, L1 = %#x, L2 = %#x", test, + (void *)ptables, l1_index, l2_index); + + uint32_t *const l1_entry = ptables + l1_index; + tr_err(&userspace_proxy_tr, "l1 @ %p = %p", (void *)l1_entry, *l1_entry); + + uint32_t* l1_ppn = dump_pte(*l1_entry); + if (!l1_ppn) { + tr_err(&userspace_proxy_tr, "INVALID L1 PTE!"); + return; + } + + uint32_t *const l2_entry = l1_ppn + l2_index; + tr_err(&userspace_proxy_tr, "l2 @ %p = %p", (void *)l2_entry, *l2_entry); + uint32_t *l2_ppn = dump_pte(*l2_entry); + + if (test_aligned != POINTER_TO_UINT(l2_ppn)) { + tr_err(&userspace_proxy_tr, "INVALID L2 PTE!"); + return; + } +} + +extern uint32_t *xtensa_kernel_ptables; + +void dump_page_tables(uint32_t *ptables, void *test, bool kernel) +{ + if (ptables) { + tr_err(&userspace_proxy_tr, "Dump for %p in user table", test); + dump_page_table(ptables, sys_cache_cached_ptr_get(test)); + dump_page_table(ptables, sys_cache_uncached_ptr_get(test)); + } + + if (kernel) { + tr_err(&userspace_proxy_tr, "Kernel table", test); + dump_page_table(xtensa_kernel_ptables, sys_cache_cached_ptr_get(test)); + dump_page_table(xtensa_kernel_ptables, sys_cache_uncached_ptr_get(test)); + } +} + +static void dump_domain_attr(char *attr_s, uint32_t attr) +{ + attr_s[0] = attr & XTENSA_MMU_MAP_SHARED ? 'S' : '-'; + attr_s[1] = K_MEM_PARTITION_IS_USER(attr) ? 'U' : '-'; + dump_pte_attr(attr_s + 2, attr); +} + +void dump_memory_domain(struct k_mem_domain *domain) +{ + int i; + char attrs[7]; + + for (i = 0; i < domain->num_partitions; i++) { + dump_domain_attr(attrs, domain->partitions[i].attr); + + tr_err(&userspace_proxy_tr, "partitions[%d]: %p + %#zx %s", i, + UINT_TO_POINTER(domain->partitions[i].start), + domain->partitions[i].size, attrs); + } + + tr_err(&userspace_proxy_tr, "ptables = %p, asid = %u, dirty = %u", + (void *)domain->arch.ptables, domain->arch.asid, domain->arch.dirty); +} + #else /* CONFIG_USERSPACE */ void *user_stack_allocate(size_t stack_size, uint32_t options)