diff --git a/posix/include/rtos/alloc.h b/posix/include/rtos/alloc.h index 294b27eaee9b..1770210cc0fb 100644 --- a/posix/include/rtos/alloc.h +++ b/posix/include/rtos/alloc.h @@ -33,19 +33,21 @@ */ /** \brief Indicates we should return DMA-able memory. */ -#define SOF_MEM_FLAG_DMA BIT(0) +#define SOF_MEM_FLAG_DMA BIT(0) /** \brief Indicates that original content should not be copied by realloc. */ -#define SOF_MEM_FLAG_NO_COPY BIT(1) +#define SOF_MEM_FLAG_NO_COPY BIT(1) /** \brief Indicates that if we should return uncached address. */ -#define SOF_MEM_FLAG_COHERENT BIT(2) +#define SOF_MEM_FLAG_COHERENT BIT(2) /** \brief Indicates that if we should return L3 address. */ -#define SOF_MEM_FLAG_L3 BIT(3) +#define SOF_MEM_FLAG_L3 BIT(3) /** \brief Indicates that if we should return Low power memory address. */ -#define SOF_MEM_FLAG_LOW_POWER BIT(4) +#define SOF_MEM_FLAG_LOW_POWER BIT(4) /** \brief Indicates that if we should return kernel memory address. */ -#define SOF_MEM_FLAG_KERNEL BIT(5) +#define SOF_MEM_FLAG_KERNEL BIT(5) /** \brief Indicates that if we should return user memory address. */ -#define SOF_MEM_FLAG_USER BIT(6) +#define SOF_MEM_FLAG_USER BIT(6) +/** \brief Indicates that if we should return shared user memory address. */ +#define SOF_MEM_FLAG_USER_SHARED_BUFFER BIT(7) /** @} */ diff --git a/posix/include/rtos/userspace_helper.h b/posix/include/rtos/userspace_helper.h new file mode 100644 index 000000000000..e535576efac7 --- /dev/null +++ b/posix/include/rtos/userspace_helper.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Jaroslaw Stelter + * Adrian Warecki + */ + +/** + * \brief Userspace support functions. + */ +#ifndef __RTOS_USERSPACE_HELPER_H__ +#define __RTOS_USERSPACE_HELPER_H__ + +#include +#include + +#include + +struct sys_heap; + +#ifdef CONFIG_USERSPACE +/** + * Initialize private processing module heap. + * @param N/A. + * @return pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * The private heap is used only for non-privileged modules for all processing module allocations + * that should be isolated. The heap helps to accumulate all dynamic allocations in single memory + * region which is then added to modules memory domain. + */ +static inline struct sys_heap *module_driver_heap_init(void) +{ + return NULL; +} + +#endif + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @param alignment - Alignment in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * @note When CONFIG_USERSPACE not set function calls rballoc_align() + */ +static inline void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, + size_t bytes, uint32_t align) +{ + return rballoc_align(flags, bytes, align); +} + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rmalloc. + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @return - Pointer to the allocated memory or NULL if failed. + * + * * @note When CONFIG_USERSPACE not set function calls rmalloc() + */ +static inline void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, + size_t bytes) +{ + return rmalloc(flags, bytes); +} + +/** + * Similar to user_rmalloc(), guarantees that returned block is zeroed. + * + * @note When CONFIG_USERSPACE not set function calls rzalloc() + */ +static inline void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, + size_t bytes) +{ + return rzalloc(flags, bytes); +} + +/** + * Frees the memory block from private module sys_heap if exists. Otherwise call rfree. + * @param ptr Pointer to the memory block. + * + * @note User should take care to not free memory allocated from sys_heap + * with module_driver_heap set to NULL. It will cause exception. + * + * When CONFIG_USERSPACE not set function calls rfree() + */ +static inline void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +{ + rfree(mem); +} + +/** + * Free private processing module heap. + * @param sys_heap pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * Frees private module heap. + */ +static inline void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +{ } + +#endif /* __RTOS_USERSPACE_HELPER_H__ */ diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index b6cbc23060e2..a71a27022e0a 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -279,17 +280,16 @@ static const struct audio_buffer_ops audio_buffer_ops = { .reset = ring_buffer_reset }; -struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, bool is_shared, +struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_available, + size_t min_free_space, bool is_shared, uint32_t id) { struct ring_buffer *ring_buffer; + int memory_flags = (is_shared ? SOF_MEM_FLAG_COHERENT : 0) | + user_get_buffer_memory_region(dev->drv); /* allocate ring_buffer structure */ - if (is_shared) - ring_buffer = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ring_buffer)); - else - ring_buffer = rzalloc(SOF_MEM_FLAG_USER, sizeof(*ring_buffer)); + ring_buffer = rzalloc(memory_flags, sizeof(*ring_buffer)); if (!ring_buffer) return NULL; @@ -359,7 +359,8 @@ struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_spa ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *) - rballoc_align(SOF_MEM_FLAG_USER, ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + rballoc_align(memory_flags, ring_buffer->data_buffer_size, + PLATFORM_DCACHE_ALIGN); if (!ring_buffer->_data_buffer) goto err; diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index da0353d892ab..1afd8c52354f 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -90,7 +90,7 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, int flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - mod = rzalloc(flags, sizeof(*mod)); + mod = module_driver_heap_rzalloc(drv->user_heap, flags, sizeof(*mod)); if (!mod) { comp_err(dev, "module_adapter_new(), failed to allocate memory for module"); goto err; @@ -210,6 +210,7 @@ int module_adapter_prepare(struct comp_dev *dev) struct list_item *blist, *_blist; uint32_t buff_periods; uint32_t buff_size; /* size of local buffer */ + int memory_flags; int i = 0; comp_dbg(dev, "module_adapter_prepare() start"); @@ -342,11 +343,11 @@ int module_adapter_prepare(struct comp_dev *dev) module_adapter_check_data(mod, dev, sink); + memory_flags = user_get_buffer_memory_region(dev->drv); /* allocate memory for input buffers */ if (mod->max_sources) { mod->input_buffers = - rzalloc(SOF_MEM_FLAG_USER, - sizeof(*mod->input_buffers) * mod->max_sources); + rzalloc(memory_flags, sizeof(*mod->input_buffers) * mod->max_sources); if (!mod->input_buffers) { comp_err(dev, "failed to allocate input buffers"); return -ENOMEM; @@ -358,8 +359,7 @@ int module_adapter_prepare(struct comp_dev *dev) /* allocate memory for output buffers */ if (mod->max_sinks) { mod->output_buffers = - rzalloc(SOF_MEM_FLAG_USER, - sizeof(*mod->output_buffers) * mod->max_sinks); + rzalloc(memory_flags, sizeof(*mod->output_buffers) * mod->max_sinks); if (!mod->output_buffers) { comp_err(dev, "failed to allocate output buffers"); ret = -ENOMEM; @@ -425,7 +425,7 @@ int module_adapter_prepare(struct comp_dev *dev) list_for_item(blist, &dev->bsource_list) { size_t size = MAX(mod->deep_buff_bytes, mod->period_bytes); - mod->input_buffers[i].data = rballoc(SOF_MEM_FLAG_USER, size); + mod->input_buffers[i].data = rballoc(memory_flags, size); if (!mod->input_buffers[i].data) { comp_err(mod->dev, "Failed to alloc input buffer data"); ret = -ENOMEM; @@ -437,7 +437,7 @@ int module_adapter_prepare(struct comp_dev *dev) /* allocate memory for output buffer data */ i = 0; list_for_item(blist, &dev->bsink_list) { - mod->output_buffers[i].data = rballoc(SOF_MEM_FLAG_USER, md->mpd.out_buff_size); + mod->output_buffers[i].data = rballoc(memory_flags, md->mpd.out_buff_size); if (!mod->output_buffers[i].data) { comp_err(mod->dev, "Failed to alloc output buffer data"); ret = -ENOMEM; @@ -450,7 +450,7 @@ int module_adapter_prepare(struct comp_dev *dev) if (list_is_empty(&mod->raw_data_buffers_list)) { for (i = 0; i < mod->num_of_sinks; i++) { /* allocate not shared buffer */ - struct comp_buffer *buffer = buffer_alloc(buff_size, SOF_MEM_FLAG_USER, + struct comp_buffer *buffer = buffer_alloc(buff_size, memory_flags, PLATFORM_DCACHE_ALIGN, false); uint32_t flags; diff --git a/src/audio/module_adapter/module_adapter_ipc4.c b/src/audio/module_adapter/module_adapter_ipc4.c index 3f0f7dc6f329..2657e7e2a561 100644 --- a/src/audio/module_adapter/module_adapter_ipc4.c +++ b/src/audio/module_adapter/module_adapter_ipc4.c @@ -136,7 +136,9 @@ int module_adapter_init_data(struct comp_dev *dev, if (cfgsz == (sizeof(*cfg) + pinsz)) { dst->nb_input_pins = n_in; dst->nb_output_pins = n_out; - dst->input_pins = rmalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, pinsz); + dst->input_pins = module_driver_heap_rmalloc(dev->drv->user_heap, + SOF_MEM_FLAG_USER | + SOF_MEM_FLAG_COHERENT, pinsz); if (!dst->input_pins) return -ENOMEM; diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 7f71af8f8025..1440a775b400 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -590,6 +591,7 @@ struct comp_driver { * Intended to replace the ops field. * Currently used by module_adapter. */ + struct sys_heap *user_heap; /**< Userspace heap */ }; /** \brief Holds constant pointer to component driver */ @@ -853,8 +855,7 @@ static inline enum sof_comp_type dev_comp_type(const struct comp_dev *dev) * @param bytes Size of the component device in bytes. * @return Pointer to the component device. */ -static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, - size_t bytes) +static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, size_t bytes) { struct comp_dev *dev = NULL; @@ -862,7 +863,8 @@ static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, * Use uncached address everywhere to access components to rule out * multi-core failures. TODO: verify if cached alias may be used in some cases */ - dev = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, bytes); + dev = module_driver_heap_rzalloc(drv->user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + bytes); if (!dev) return NULL; dev->size = bytes; @@ -1192,4 +1194,12 @@ void comp_init_performance_data(struct comp_dev *dev); */ bool comp_update_performance_data(struct comp_dev *dev, uint32_t cycles_used); +static inline int user_get_buffer_memory_region(const struct comp_driver *drv) +{ +#if CONFIG_USERSPACE + return drv->user_heap ? SOF_MEM_FLAG_USER_SHARED_BUFFER : SOF_MEM_FLAG_USER; +#else + return SOF_MEM_FLAG_USER; +#endif +} #endif /* __SOF_AUDIO_COMPONENT_H__ */ diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 1001c4ba450c..3a30b97172eb 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -163,6 +163,26 @@ int module_prepare(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks); +static inline +bool generic_module_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, + int num_of_sources, + struct sof_sink **sinks, + int num_of_sinks) +{ + int i; + + for (i = 0; i < num_of_sources; i++) + if (source_get_data_available(sources[i]) < source_get_min_available(sources[i])) + return false; + + for (i = 0; i < num_of_sinks; i++) + if (sink_get_free_size(sinks[i]) < sink_get_min_free_space(sinks[i])) + return false; + + return true; +} + static inline bool module_is_ready_to_process(struct processing_module *mod, struct sof_source **sources, @@ -170,7 +190,6 @@ bool module_is_ready_to_process(struct processing_module *mod, struct sof_sink **sinks, int num_of_sinks) { - int i; const struct module_interface *const ops = mod->dev->drv->adapter_ops; /* LL module has to be always ready for processing */ @@ -182,15 +201,8 @@ bool module_is_ready_to_process(struct processing_module *mod, /* default action - the module is ready if there's enough data for processing and enough * space to store result. IBS/OBS as declared in init_instance */ - for (i = 0; i < num_of_sources; i++) - if (source_get_data_available(sources[i]) < source_get_min_available(sources[i])) - return false; - - for (i = 0; i < num_of_sinks; i++) - if (sink_get_free_size(sinks[i]) < sink_get_min_free_space(sinks[i])) - return false; - - return true; + return generic_module_is_ready_to_process(mod, sources, num_of_sources, sinks, + num_of_sinks); } int module_process_sink_src(struct processing_module *mod, diff --git a/src/include/sof/audio/ring_buffer.h b/src/include/sof/audio/ring_buffer.h index 222b102f2ae9..1ef712a56743 100644 --- a/src/include/sof/audio/ring_buffer.h +++ b/src/include/sof/audio/ring_buffer.h @@ -97,6 +97,7 @@ * always means "buffer full" */ +struct comp_dev; struct ring_buffer; struct sof_audio_stream_params; @@ -114,6 +115,7 @@ struct ring_buffer { /** * + * @param dev pointer to the DP module device structure * @param min_available minimum data available in queue required by the module using * ring_buffer's source api * @param min_free_space minimum buffer space in queue required by the module using @@ -122,7 +124,8 @@ struct ring_buffer { * @param id a stream ID, accessible later by sink_get_id/source_get_id * */ -struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, bool is_shared, +struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_available, + size_t min_free_space, bool is_shared, uint32_t id); #endif /* __SOF_RING_BUFFER_H__ */ diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index f9b7f16c0b0c..ef32b5d75e87 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -186,6 +186,18 @@ int lib_manager_register_module(const uint32_t component_id); */ const struct sof_man_fw_desc *lib_manager_get_library_manifest(int module_id); +/* + * \brief Get address and size of the bss section for given module instance + * + * param[in] instance_id - instance id + * param[in] mod - module manifest + * param[out] va_addr - address of the bss section + * param[out] size - size of the bss section + */ +void lib_manager_get_instance_bss_address(uint32_t instance_id, + const struct sof_man_module *mod, + void __sparse_cache **va_addr, size_t *size); + /* * \brief Free module * diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 4e2f54db3553..511917ceb768 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -603,25 +603,23 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP || source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { - struct sof_source *source = audio_buffer_get_source(&buffer->audio_buffer); - struct sof_sink *sink = audio_buffer_get_sink(&buffer->audio_buffer); + bool dp_on_source = source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP; + struct sof_source *src = audio_buffer_get_source(&buffer->audio_buffer); + struct sof_sink *snk = audio_buffer_get_sink(&buffer->audio_buffer); - ring_buffer = ring_buffer_create(source_get_min_available(source), - sink_get_min_free_space(sink), + ring_buffer = ring_buffer_create(dp_on_source ? source : sink, + source_get_min_available(src), + sink_get_min_free_space(snk), audio_buffer_is_shared(&buffer->audio_buffer), buf_get_id(buffer)); if (!ring_buffer) goto free; - } - if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) /* data destination module needs to use ring_buffer */ - audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, false, /* at_input */ - &ring_buffer->audio_buffer); - else if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - /* data source module needs to use ring_buffer */ - audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, true, /* at_input */ + audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, dp_on_source, &ring_buffer->audio_buffer); + } + #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ /* * Connect and bind the buffer to both source and sink components with LL processing been diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index c153c61a308b..6170bbe1ccdc 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -273,9 +273,9 @@ static int lib_manager_unload_libcode_modules(const uint32_t module_id) } #endif /* CONFIG_LIBCODE_MODULE_SUPPORT */ -static void lib_manager_get_instance_bss_address(uint32_t instance_id, - const struct sof_man_module *mod, - void __sparse_cache **va_addr, size_t *size) +void lib_manager_get_instance_bss_address(uint32_t instance_id, + const struct sof_man_module *mod, + void __sparse_cache **va_addr, size_t *size) { *size = mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length / mod->instance_max_count * PAGE_SZ; diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index f0dd442af0c1..de22a83f574e 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -474,6 +474,7 @@ zephyr_library_sources( lib/alloc.c lib/cpu.c lib/pm_runtime.c + lib/userspace_helper.c # Common library functions - Will be moved to Zephyr over time lib.c @@ -521,6 +522,10 @@ if (NOT CONFIG_COMPILER_INLINE_FUNCTION_OPTION) target_compile_options(SOF INTERFACE -fno-inline-functions) endif() +if (CONFIG_USERSPACE) +target_compile_options(SOF INTERFACE -mno-global-merge) +endif() + # SOF needs `typeof`, `__VA_ARGS__` and maybe other GNU C99 # extensions. TODO other flags required ? target_compile_options(SOF INTERFACE $<$: -std=gnu99>) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index d034cd5aa1db..8d8b6f3198a0 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -41,6 +41,18 @@ config SOF_ZEPHYR_HEAP_SIZE NOTE: Keep in mind that the heap size should not be greater than the physical memory size of the system defined in DT (and this includes baseFW text/data). +config SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE + hex "Size of the shared buffer heap for SOF userspace modules" + default 0x1E000 if SOC_INTEL_ACE15_MTPM || SOC_INTEL_ACE20_LNL + default 0x1A000 if SOC_INTEL_ACE30 + default 0x0 + help + The size of the zephyr heap portion designated as the shared buffer heap. + This heap is shared between the sof and userspace modules. It is used exclusively for + allocating audio buffers between the kernel and userspace modules. + NOTE: Keep in mind that the shared heap size should not be greater than the + heap size. + config SOF_ZEPHYR_VIRTUAL_HEAP_SIZE hex "Size of the Zephyr virtual heap for SOF" depends on VIRTUAL_HEAP @@ -57,6 +69,14 @@ config SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE help This config defines size of virtual heap region shared between all cores +config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE + hex "Size of the private heap created for each userspace module" + default 0x1000 + help + The size of the private heap created for each userspace module. Each userspace + module has its own independent heap to which only it has access. This heap is + shared between instances of the same module. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" default n diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 600852418693..d590b3527626 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -24,19 +24,21 @@ */ /** \brief Indicates we should return DMA-able memory. */ -#define SOF_MEM_FLAG_DMA BIT(0) +#define SOF_MEM_FLAG_DMA BIT(0) /** \brief Indicates that original content should not be copied by realloc. */ -#define SOF_MEM_FLAG_NO_COPY BIT(1) +#define SOF_MEM_FLAG_NO_COPY BIT(1) /** \brief Indicates that if we should return uncached address. */ -#define SOF_MEM_FLAG_COHERENT BIT(2) +#define SOF_MEM_FLAG_COHERENT BIT(2) /** \brief Indicates that if we should return L3 address. */ -#define SOF_MEM_FLAG_L3 BIT(3) +#define SOF_MEM_FLAG_L3 BIT(3) /** \brief Indicates that if we should return Low power memory address. */ -#define SOF_MEM_FLAG_LOW_POWER BIT(4) +#define SOF_MEM_FLAG_LOW_POWER BIT(4) /** \brief Indicates that if we should return kernel memory address. */ -#define SOF_MEM_FLAG_KERNEL BIT(5) +#define SOF_MEM_FLAG_KERNEL BIT(5) /** \brief Indicates that if we should return user memory address. */ -#define SOF_MEM_FLAG_USER BIT(6) +#define SOF_MEM_FLAG_USER BIT(6) +/** \brief Indicates that if we should return shared user memory address. */ +#define SOF_MEM_FLAG_USER_SHARED_BUFFER BIT(7) /** @} */ @@ -111,4 +113,21 @@ static inline void heap_trace_all(int force) {} /** @}*/ +#if CONFIG_USERSPACE +/** + * Returns the start address of shared memory heap for buffers. + * + * @return pointer to shared memory heap start + */ +uintptr_t get_shared_buffer_heap_start(void); + +/** + * Returns the size of shared memory heap for buffers. + * + * @return size of shared memory heap start + */ +size_t get_shared_buffer_heap_size(void); + +#endif + #endif /* __ZEPHYR_RTOS_ALLOC_H__ */ diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h new file mode 100644 index 000000000000..1e7af9ad3773 --- /dev/null +++ b/zephyr/include/rtos/userspace_helper.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Jaroslaw Stelter + * Adrian Warecki + */ + +/** + * \brief Userspace support functions. + */ +#ifndef __ZEPHYR_LIB_USERSPACE_HELPER_H__ +#define __ZEPHYR_LIB_USERSPACE_HELPER_H__ + +#ifdef CONFIG_USERSPACE +#define DRV_HEAP_SIZE ALIGN_UP(CONFIG_SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE, \ + CONFIG_MM_DRV_PAGE_SIZE) + +/** + * Initialize private processing module heap. + * @param N/A. + * @return pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * The private heap is used only for non-privileged modules for all processing module allocations + * that should be isolated. The heap helps to accumulate all dynamic allocations in single memory + * region which is then added to modules memory domain. + */ +struct sys_heap *module_driver_heap_init(void); + +#endif + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @param alignment - Alignment in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * @note When CONFIG_USERSPACE not set function calls rballoc_align() + */ +void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, + uint32_t align); + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rmalloc. + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @return - Pointer to the allocated memory or NULL if failed. + * + * * @note When CONFIG_USERSPACE not set function calls rmalloc() + */ +void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes); + +/** + * Similar to user_rmalloc(), guarantees that returned block is zeroed. + * + * @note When CONFIG_USERSPACE not set function calls rzalloc() + */ +void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes); + +/** + * Frees the memory block from private module sys_heap if exists. Otherwise call rfree. + * @param ptr Pointer to the memory block. + * + * @note User should take care to not free memory allocated from sys_heap + * with mod_drv_heap set to NULL. It will cause exception. + * + * When CONFIG_USERSPACE not set function calls rfree() + */ +void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem); + +/** + * Free private processing module heap. + * @param sys_heap pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * Frees private module heap. + */ +void module_driver_heap_remove(struct sys_heap *mod_drv_heap); + +#endif /* __ZEPHYR_LIB_USERSPACE_HELPER_H__ */ diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index ff5551881125..9a55a38208ad 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -19,6 +19,8 @@ #include #include +#define SHARED_BUFFER_HEAP_MEM_SIZE 0 + #if CONFIG_VIRTUAL_HEAP #include #include @@ -99,7 +101,15 @@ static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE]; * to allow memory management driver to control unused * memory pages. */ -__section(".heap_mem") static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE]; +#if CONFIG_USERSPACE +#undef SHARED_BUFFER_HEAP_MEM_SIZE +#define SHARED_BUFFER_HEAP_MEM_SIZE ROUND_UP(CONFIG_SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE, \ + HOST_PAGE_SIZE) +__section(".shared_heap_mem") +static uint8_t __aligned(HOST_PAGE_SIZE) shared_heapmem[SHARED_BUFFER_HEAP_MEM_SIZE]; +#endif /* CONFIG_USERSPACE */ +__section(".heap_mem") +static uint8_t __aligned(HOST_PAGE_SIZE) heapmem[HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE]; #elif defined(CONFIG_ARCH_POSIX) @@ -126,6 +136,40 @@ extern char _end[], _heap_sentry[]; static struct k_heap sof_heap; +#if CONFIG_USERSPACE +static struct k_heap shared_buffer_heap; + +static bool is_shared_buffer_heap_pointer(void *ptr) +{ + uintptr_t shd_heap_start = POINTER_TO_UINT(shared_heapmem); + uintptr_t shd_heap_end = POINTER_TO_UINT(shared_heapmem + SHARED_BUFFER_HEAP_MEM_SIZE); + + if (is_cached(ptr)) + ptr = sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)ptr); + + return (POINTER_TO_UINT(ptr) >= shd_heap_start) && (POINTER_TO_UINT(ptr) < shd_heap_end); +} + +/** + * Returns the start of HPSRAM Shared memory heap. + * @return Pointer to the HPSRAM Shared memory location which can be used + * for HPSRAM Shared heap. + */ +uintptr_t get_shared_buffer_heap_start(void) +{ + return ROUND_UP(POINTER_TO_UINT(shared_heapmem), HOST_PAGE_SIZE); +} + +/** + * Returns the size of HPSRAM Shared memory heap. + * @return Size of the HPSRAM Shared memory region which can be used for HPSRAM Shared heap. + */ +size_t get_shared_buffer_heap_size(void) +{ + return ROUND_DOWN(SHARED_BUFFER_HEAP_MEM_SIZE, HOST_PAGE_SIZE); +} +#endif /* CONFIG_USERSPACE */ + #if CONFIG_L3_HEAP static struct k_heap l3_heap; static struct k_heap l3_heap_copy __imrdata; @@ -244,6 +288,8 @@ static void *virtual_heap_alloc(struct vmh_heap *heap, uint32_t flags, size_t by return mem; } +extern int _unused_ram_start_marker; + /** * Checks whether pointer is from virtual memory range. * @param ptr Pointer to memory being checked. @@ -251,8 +297,8 @@ static void *virtual_heap_alloc(struct vmh_heap *heap, uint32_t flags, size_t by */ static bool is_virtual_heap_pointer(void *ptr) { - uintptr_t virtual_heap_start = POINTER_TO_UINT(sys_cache_cached_ptr_get(&heapmem)) + - HEAPMEM_SIZE; + uintptr_t virtual_heap_start = + POINTER_TO_UINT(sys_cache_cached_ptr_get(&_unused_ram_start_marker)); uintptr_t virtual_heap_end = CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_SIZE; if (!is_cached(ptr)) @@ -425,6 +471,10 @@ void *rmalloc(uint32_t flags, size_t bytes) return ptr; #else k_panic(); +#endif +#if CONFIG_USERSPACE + } else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) { + heap = &shared_buffer_heap; #endif } else { heap = &sof_heap; @@ -509,16 +559,20 @@ void *rballoc_align(uint32_t flags, size_t bytes, tr_err(&zephyr_tr, "L3_HEAP not available."); return NULL; #endif +#if CONFIG_USERSPACE + } else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) { + heap = &shared_buffer_heap; +#endif /* CONFIG_USERSPACE */ } else { - heap = &sof_heap; - } - #if CONFIG_VIRTUAL_HEAP /* Use virtual heap if it is available */ if (virtual_buffers_heap) return virtual_heap_alloc(virtual_buffers_heap, flags, bytes, align); #endif /* CONFIG_VIRTUAL_HEAP */ + heap = &sof_heap; + } + if (flags & SOF_MEM_FLAG_COHERENT) return heap_alloc_aligned(heap, align, bytes); @@ -548,13 +602,24 @@ void rfree(void *ptr) } #endif +#if CONFIG_USERSPACE + if (is_shared_buffer_heap_pointer(ptr)) { + heap_free(&shared_buffer_heap, ptr); + return; + } +#endif + heap_free(&sof_heap, ptr); } EXPORT_SYMBOL(rfree); static int heap_init(void) { - sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE); + sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE); + +#if CONFIG_USERSPACE + sys_heap_init(&shared_buffer_heap.heap, shared_heapmem, SHARED_BUFFER_HEAP_MEM_SIZE); +#endif #if CONFIG_L3_HEAP if (l3_heap_copy.heap.heap) diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c new file mode 100644 index 000000000000..74698d7712c1 --- /dev/null +++ b/zephyr/lib/userspace_helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2025 Intel Corporation. All rights reserved. +// +// Author: Jaroslaw Stelter +// Adrian Warecki + +/** + * \file + * \brief Zephyr userspace helper functions + * \authors Jaroslaw Stelter + * \authors Adrian Warecki + */ + +#include + +#include +#include + +#define MODULE_DRIVER_HEAP_CACHED CONFIG_SOF_ZEPHYR_HEAP_CACHED + +#if CONFIG_USERSPACE +struct sys_heap *module_driver_heap_init(void) +{ + struct sys_heap *mod_drv_heap = rballoc(SOF_MEM_FLAG_USER, sizeof(struct sys_heap)); + + if (!mod_drv_heap) + return NULL; + + void *mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, DRV_HEAP_SIZE, + CONFIG_MM_DRV_PAGE_SIZE); + if (!mem) { + rfree(mod_drv_heap); + return NULL; + } + + sys_heap_init(mod_drv_heap, mem, DRV_HEAP_SIZE); + mod_drv_heap->init_mem = mem; + mod_drv_heap->init_bytes = DRV_HEAP_SIZE; + + return mod_drv_heap; +} + +void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, + uint32_t align) +{ +#ifdef MODULE_DRIVER_HEAP_CACHED + const bool cached = (flags & SOF_MEM_FLAG_COHERENT) == 0; +#endif /* MODULE_DRIVER_HEAP_CACHED */ + + if (mod_drv_heap) { +#ifdef MODULE_DRIVER_HEAP_CACHED + if (cached) { + /* + * Zephyr sys_heap stores metadata at start of each + * heap allocation. To ensure no allocated cached buffer + * overlaps the same cacheline with the metadata chunk, + * align both allocation start and size of allocation + * to cacheline. As cached and non-cached allocations are + * mixed, same rules need to be followed for both type of + * allocations. + */ + align = MAX(PLATFORM_DCACHE_ALIGN, align); + bytes = ALIGN_UP(bytes, align); + } +#endif /* MODULE_DRIVER_HEAP_CACHED */ + void *mem = sys_heap_aligned_alloc(mod_drv_heap, align, bytes); +#ifdef MODULE_DRIVER_HEAP_CACHED + if (cached) + return sys_cache_cached_ptr_get(mem); +#endif /* MODULE_DRIVER_HEAP_CACHED */ + return mem; + } else { + return rballoc_align(flags, bytes, align); + } +} + +void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + if (mod_drv_heap) + return module_driver_heap_aligned_alloc(mod_drv_heap, flags, bytes, 0); + else + return rmalloc(flags, bytes); +} + +void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + void *ptr; + + ptr = module_driver_heap_rmalloc(mod_drv_heap, flags, bytes); + if (ptr) + memset(ptr, 0, bytes); + + return ptr; +} + +void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +{ + if (mod_drv_heap) { +#ifdef MODULE_DRIVER_HEAP_CACHED + if (is_cached(mem)) { + void *mem_uncached = sys_cache_uncached_ptr_get( + (__sparse_force void __sparse_cache *)mem); + + sys_cache_data_invd_range(mem, + sys_heap_usable_size(mod_drv_heap, mem_uncached)); + + mem = mem_uncached; + } +#endif + sys_heap_free(mod_drv_heap, mem); + } else { + rfree(mem); + } +} + +void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +{ + if (mod_drv_heap) { + rfree(mod_drv_heap->init_mem); + rfree(mod_drv_heap); + } +} + +#else /* CONFIG_USERSPACE */ + +void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + return rmalloc(flags, bytes); +} + +void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, + uint32_t align) +{ + return rballoc_align(flags, bytes, align); +} + +void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + return rzalloc(flags, bytes); +} + +void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +{ + rfree(mem); +} + +void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +{ } + +#endif /* CONFIG_USERSPACE */