From d581d364277d8d92c515782bf8aa0f4bd5925050 Mon Sep 17 00:00:00 2001 From: Jaroslaw Stelter Date: Tue, 29 Aug 2023 12:39:16 +0200 Subject: [PATCH 1/7] alloc: userspace: Introduce MMU shared memory heap In case of CONFIG_USERSPACE enabled, all memory is accessible form kernel space. For accessing it from user space we need to grant access to its explicitly. However there are some infrastructure components that could be shared between kernel and user spaces. This patch splits system heap into two sections. One section which will contatin shareable data will be located in separate memory partition. Each non-privilidged module will obtain it in its private memory domain. Signed-off-by: Jaroslaw Stelter Signed-off-by: Adrian Warecki --- posix/include/rtos/alloc.h | 16 ++++---- zephyr/Kconfig | 12 ++++++ zephyr/include/rtos/alloc.h | 33 ++++++++++++---- zephyr/lib/alloc.c | 79 +++++++++++++++++++++++++++++++++---- 4 files changed, 119 insertions(+), 21 deletions(-) diff --git a/posix/include/rtos/alloc.h b/posix/include/rtos/alloc.h index 294b27eaee9b..1770210cc0fb 100644 --- a/posix/include/rtos/alloc.h +++ b/posix/include/rtos/alloc.h @@ -33,19 +33,21 @@ */ /** \brief Indicates we should return DMA-able memory. */ -#define SOF_MEM_FLAG_DMA BIT(0) +#define SOF_MEM_FLAG_DMA BIT(0) /** \brief Indicates that original content should not be copied by realloc. */ -#define SOF_MEM_FLAG_NO_COPY BIT(1) +#define SOF_MEM_FLAG_NO_COPY BIT(1) /** \brief Indicates that if we should return uncached address. */ -#define SOF_MEM_FLAG_COHERENT BIT(2) +#define SOF_MEM_FLAG_COHERENT BIT(2) /** \brief Indicates that if we should return L3 address. */ -#define SOF_MEM_FLAG_L3 BIT(3) +#define SOF_MEM_FLAG_L3 BIT(3) /** \brief Indicates that if we should return Low power memory address. */ -#define SOF_MEM_FLAG_LOW_POWER BIT(4) +#define SOF_MEM_FLAG_LOW_POWER BIT(4) /** \brief Indicates that if we should return kernel memory address. */ -#define SOF_MEM_FLAG_KERNEL BIT(5) +#define SOF_MEM_FLAG_KERNEL BIT(5) /** \brief Indicates that if we should return user memory address. */ -#define SOF_MEM_FLAG_USER BIT(6) +#define SOF_MEM_FLAG_USER BIT(6) +/** \brief Indicates that if we should return shared user memory address. */ +#define SOF_MEM_FLAG_USER_SHARED_BUFFER BIT(7) /** @} */ diff --git a/zephyr/Kconfig b/zephyr/Kconfig index d034cd5aa1db..060cc8be2720 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -41,6 +41,18 @@ config SOF_ZEPHYR_HEAP_SIZE NOTE: Keep in mind that the heap size should not be greater than the physical memory size of the system defined in DT (and this includes baseFW text/data). +config SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE + hex "Size of the shared buffer heap for SOF userspace modules" + default 0x1E000 if SOC_INTEL_ACE15_MTPM || SOC_INTEL_ACE20_LNL + default 0x1A000 if SOC_INTEL_ACE30 + default 0x0 + help + The size of the zephyr heap portion designated as the shared buffer heap. + This heap is shared between the sof and userspace modules. It is used exclusively for + allocating audio buffers between the kernel and userspace modules. + NOTE: Keep in mind that the shared heap size should not be greater than the + heap size. + config SOF_ZEPHYR_VIRTUAL_HEAP_SIZE hex "Size of the Zephyr virtual heap for SOF" depends on VIRTUAL_HEAP diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 600852418693..d590b3527626 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -24,19 +24,21 @@ */ /** \brief Indicates we should return DMA-able memory. */ -#define SOF_MEM_FLAG_DMA BIT(0) +#define SOF_MEM_FLAG_DMA BIT(0) /** \brief Indicates that original content should not be copied by realloc. */ -#define SOF_MEM_FLAG_NO_COPY BIT(1) +#define SOF_MEM_FLAG_NO_COPY BIT(1) /** \brief Indicates that if we should return uncached address. */ -#define SOF_MEM_FLAG_COHERENT BIT(2) +#define SOF_MEM_FLAG_COHERENT BIT(2) /** \brief Indicates that if we should return L3 address. */ -#define SOF_MEM_FLAG_L3 BIT(3) +#define SOF_MEM_FLAG_L3 BIT(3) /** \brief Indicates that if we should return Low power memory address. */ -#define SOF_MEM_FLAG_LOW_POWER BIT(4) +#define SOF_MEM_FLAG_LOW_POWER BIT(4) /** \brief Indicates that if we should return kernel memory address. */ -#define SOF_MEM_FLAG_KERNEL BIT(5) +#define SOF_MEM_FLAG_KERNEL BIT(5) /** \brief Indicates that if we should return user memory address. */ -#define SOF_MEM_FLAG_USER BIT(6) +#define SOF_MEM_FLAG_USER BIT(6) +/** \brief Indicates that if we should return shared user memory address. */ +#define SOF_MEM_FLAG_USER_SHARED_BUFFER BIT(7) /** @} */ @@ -111,4 +113,21 @@ static inline void heap_trace_all(int force) {} /** @}*/ +#if CONFIG_USERSPACE +/** + * Returns the start address of shared memory heap for buffers. + * + * @return pointer to shared memory heap start + */ +uintptr_t get_shared_buffer_heap_start(void); + +/** + * Returns the size of shared memory heap for buffers. + * + * @return size of shared memory heap start + */ +size_t get_shared_buffer_heap_size(void); + +#endif + #endif /* __ZEPHYR_RTOS_ALLOC_H__ */ diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index ff5551881125..9a55a38208ad 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -19,6 +19,8 @@ #include #include +#define SHARED_BUFFER_HEAP_MEM_SIZE 0 + #if CONFIG_VIRTUAL_HEAP #include #include @@ -99,7 +101,15 @@ static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE]; * to allow memory management driver to control unused * memory pages. */ -__section(".heap_mem") static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE]; +#if CONFIG_USERSPACE +#undef SHARED_BUFFER_HEAP_MEM_SIZE +#define SHARED_BUFFER_HEAP_MEM_SIZE ROUND_UP(CONFIG_SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE, \ + HOST_PAGE_SIZE) +__section(".shared_heap_mem") +static uint8_t __aligned(HOST_PAGE_SIZE) shared_heapmem[SHARED_BUFFER_HEAP_MEM_SIZE]; +#endif /* CONFIG_USERSPACE */ +__section(".heap_mem") +static uint8_t __aligned(HOST_PAGE_SIZE) heapmem[HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE]; #elif defined(CONFIG_ARCH_POSIX) @@ -126,6 +136,40 @@ extern char _end[], _heap_sentry[]; static struct k_heap sof_heap; +#if CONFIG_USERSPACE +static struct k_heap shared_buffer_heap; + +static bool is_shared_buffer_heap_pointer(void *ptr) +{ + uintptr_t shd_heap_start = POINTER_TO_UINT(shared_heapmem); + uintptr_t shd_heap_end = POINTER_TO_UINT(shared_heapmem + SHARED_BUFFER_HEAP_MEM_SIZE); + + if (is_cached(ptr)) + ptr = sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)ptr); + + return (POINTER_TO_UINT(ptr) >= shd_heap_start) && (POINTER_TO_UINT(ptr) < shd_heap_end); +} + +/** + * Returns the start of HPSRAM Shared memory heap. + * @return Pointer to the HPSRAM Shared memory location which can be used + * for HPSRAM Shared heap. + */ +uintptr_t get_shared_buffer_heap_start(void) +{ + return ROUND_UP(POINTER_TO_UINT(shared_heapmem), HOST_PAGE_SIZE); +} + +/** + * Returns the size of HPSRAM Shared memory heap. + * @return Size of the HPSRAM Shared memory region which can be used for HPSRAM Shared heap. + */ +size_t get_shared_buffer_heap_size(void) +{ + return ROUND_DOWN(SHARED_BUFFER_HEAP_MEM_SIZE, HOST_PAGE_SIZE); +} +#endif /* CONFIG_USERSPACE */ + #if CONFIG_L3_HEAP static struct k_heap l3_heap; static struct k_heap l3_heap_copy __imrdata; @@ -244,6 +288,8 @@ static void *virtual_heap_alloc(struct vmh_heap *heap, uint32_t flags, size_t by return mem; } +extern int _unused_ram_start_marker; + /** * Checks whether pointer is from virtual memory range. * @param ptr Pointer to memory being checked. @@ -251,8 +297,8 @@ static void *virtual_heap_alloc(struct vmh_heap *heap, uint32_t flags, size_t by */ static bool is_virtual_heap_pointer(void *ptr) { - uintptr_t virtual_heap_start = POINTER_TO_UINT(sys_cache_cached_ptr_get(&heapmem)) + - HEAPMEM_SIZE; + uintptr_t virtual_heap_start = + POINTER_TO_UINT(sys_cache_cached_ptr_get(&_unused_ram_start_marker)); uintptr_t virtual_heap_end = CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_SIZE; if (!is_cached(ptr)) @@ -425,6 +471,10 @@ void *rmalloc(uint32_t flags, size_t bytes) return ptr; #else k_panic(); +#endif +#if CONFIG_USERSPACE + } else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) { + heap = &shared_buffer_heap; #endif } else { heap = &sof_heap; @@ -509,16 +559,20 @@ void *rballoc_align(uint32_t flags, size_t bytes, tr_err(&zephyr_tr, "L3_HEAP not available."); return NULL; #endif +#if CONFIG_USERSPACE + } else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) { + heap = &shared_buffer_heap; +#endif /* CONFIG_USERSPACE */ } else { - heap = &sof_heap; - } - #if CONFIG_VIRTUAL_HEAP /* Use virtual heap if it is available */ if (virtual_buffers_heap) return virtual_heap_alloc(virtual_buffers_heap, flags, bytes, align); #endif /* CONFIG_VIRTUAL_HEAP */ + heap = &sof_heap; + } + if (flags & SOF_MEM_FLAG_COHERENT) return heap_alloc_aligned(heap, align, bytes); @@ -548,13 +602,24 @@ void rfree(void *ptr) } #endif +#if CONFIG_USERSPACE + if (is_shared_buffer_heap_pointer(ptr)) { + heap_free(&shared_buffer_heap, ptr); + return; + } +#endif + heap_free(&sof_heap, ptr); } EXPORT_SYMBOL(rfree); static int heap_init(void) { - sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE); + sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE); + +#if CONFIG_USERSPACE + sys_heap_init(&shared_buffer_heap.heap, shared_heapmem, SHARED_BUFFER_HEAP_MEM_SIZE); +#endif #if CONFIG_L3_HEAP if (l3_heap_copy.heap.heap) From 44705a860ccadd486e1481b1a91a8c09c8955ee9 Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Mon, 7 Jul 2025 20:36:54 +0200 Subject: [PATCH 2/7] ptl: mmu: Introduce module driver heap for non-privileged modules Non-privileged modules should be isolated each other. It means that module related data should be located in memory available for kernel and the owner module only. To manage access all such data should be allocated from single, separate region. This patch introduces private module heap. It is allocated from regular heap, assigned to module thread and then used to allocate all infrastructure data associated with the module. Other non-privileged modules will not have access to this region. Currently it is arbitrary defined to use 5 pages, what could change. Signed-off-by: Jaroslaw Stelter Signed-off-by: Adrian Warecki --- posix/include/rtos/userspace_helper.h | 108 +++++++++++++ src/audio/module_adapter/module_adapter.c | 2 +- .../module_adapter/module_adapter_ipc4.c | 4 +- src/include/sof/audio/component.h | 8 +- zephyr/CMakeLists.txt | 1 + zephyr/Kconfig | 8 + zephyr/include/rtos/userspace_helper.h | 86 ++++++++++ zephyr/lib/userspace_helper.c | 151 ++++++++++++++++++ 8 files changed, 363 insertions(+), 5 deletions(-) create mode 100644 posix/include/rtos/userspace_helper.h create mode 100644 zephyr/include/rtos/userspace_helper.h create mode 100644 zephyr/lib/userspace_helper.c diff --git a/posix/include/rtos/userspace_helper.h b/posix/include/rtos/userspace_helper.h new file mode 100644 index 000000000000..e535576efac7 --- /dev/null +++ b/posix/include/rtos/userspace_helper.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Jaroslaw Stelter + * Adrian Warecki + */ + +/** + * \brief Userspace support functions. + */ +#ifndef __RTOS_USERSPACE_HELPER_H__ +#define __RTOS_USERSPACE_HELPER_H__ + +#include +#include + +#include + +struct sys_heap; + +#ifdef CONFIG_USERSPACE +/** + * Initialize private processing module heap. + * @param N/A. + * @return pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * The private heap is used only for non-privileged modules for all processing module allocations + * that should be isolated. The heap helps to accumulate all dynamic allocations in single memory + * region which is then added to modules memory domain. + */ +static inline struct sys_heap *module_driver_heap_init(void) +{ + return NULL; +} + +#endif + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @param alignment - Alignment in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * @note When CONFIG_USERSPACE not set function calls rballoc_align() + */ +static inline void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, + size_t bytes, uint32_t align) +{ + return rballoc_align(flags, bytes, align); +} + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rmalloc. + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @return - Pointer to the allocated memory or NULL if failed. + * + * * @note When CONFIG_USERSPACE not set function calls rmalloc() + */ +static inline void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, + size_t bytes) +{ + return rmalloc(flags, bytes); +} + +/** + * Similar to user_rmalloc(), guarantees that returned block is zeroed. + * + * @note When CONFIG_USERSPACE not set function calls rzalloc() + */ +static inline void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, + size_t bytes) +{ + return rzalloc(flags, bytes); +} + +/** + * Frees the memory block from private module sys_heap if exists. Otherwise call rfree. + * @param ptr Pointer to the memory block. + * + * @note User should take care to not free memory allocated from sys_heap + * with module_driver_heap set to NULL. It will cause exception. + * + * When CONFIG_USERSPACE not set function calls rfree() + */ +static inline void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +{ + rfree(mem); +} + +/** + * Free private processing module heap. + * @param sys_heap pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * Frees private module heap. + */ +static inline void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +{ } + +#endif /* __RTOS_USERSPACE_HELPER_H__ */ diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index da0353d892ab..e2a80f0fa9e3 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -90,7 +90,7 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, int flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - mod = rzalloc(flags, sizeof(*mod)); + mod = module_driver_heap_rzalloc(drv->user_heap, flags, sizeof(*mod)); if (!mod) { comp_err(dev, "module_adapter_new(), failed to allocate memory for module"); goto err; diff --git a/src/audio/module_adapter/module_adapter_ipc4.c b/src/audio/module_adapter/module_adapter_ipc4.c index 3f0f7dc6f329..2657e7e2a561 100644 --- a/src/audio/module_adapter/module_adapter_ipc4.c +++ b/src/audio/module_adapter/module_adapter_ipc4.c @@ -136,7 +136,9 @@ int module_adapter_init_data(struct comp_dev *dev, if (cfgsz == (sizeof(*cfg) + pinsz)) { dst->nb_input_pins = n_in; dst->nb_output_pins = n_out; - dst->input_pins = rmalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, pinsz); + dst->input_pins = module_driver_heap_rmalloc(dev->drv->user_heap, + SOF_MEM_FLAG_USER | + SOF_MEM_FLAG_COHERENT, pinsz); if (!dst->input_pins) return -ENOMEM; diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 7f71af8f8025..40e43ef7c8a4 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -590,6 +591,7 @@ struct comp_driver { * Intended to replace the ops field. * Currently used by module_adapter. */ + struct sys_heap *user_heap; /**< Userspace heap */ }; /** \brief Holds constant pointer to component driver */ @@ -853,8 +855,7 @@ static inline enum sof_comp_type dev_comp_type(const struct comp_dev *dev) * @param bytes Size of the component device in bytes. * @return Pointer to the component device. */ -static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, - size_t bytes) +static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, size_t bytes) { struct comp_dev *dev = NULL; @@ -862,7 +863,8 @@ static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, * Use uncached address everywhere to access components to rule out * multi-core failures. TODO: verify if cached alias may be used in some cases */ - dev = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, bytes); + dev = module_driver_heap_rzalloc(drv->user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + bytes); if (!dev) return NULL; dev->size = bytes; diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index f0dd442af0c1..6edc4246c63f 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -474,6 +474,7 @@ zephyr_library_sources( lib/alloc.c lib/cpu.c lib/pm_runtime.c + lib/userspace_helper.c # Common library functions - Will be moved to Zephyr over time lib.c diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 060cc8be2720..8d8b6f3198a0 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -69,6 +69,14 @@ config SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE help This config defines size of virtual heap region shared between all cores +config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE + hex "Size of the private heap created for each userspace module" + default 0x1000 + help + The size of the private heap created for each userspace module. Each userspace + module has its own independent heap to which only it has access. This heap is + shared between instances of the same module. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" default n diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h new file mode 100644 index 000000000000..1e7af9ad3773 --- /dev/null +++ b/zephyr/include/rtos/userspace_helper.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Jaroslaw Stelter + * Adrian Warecki + */ + +/** + * \brief Userspace support functions. + */ +#ifndef __ZEPHYR_LIB_USERSPACE_HELPER_H__ +#define __ZEPHYR_LIB_USERSPACE_HELPER_H__ + +#ifdef CONFIG_USERSPACE +#define DRV_HEAP_SIZE ALIGN_UP(CONFIG_SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE, \ + CONFIG_MM_DRV_PAGE_SIZE) + +/** + * Initialize private processing module heap. + * @param N/A. + * @return pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * The private heap is used only for non-privileged modules for all processing module allocations + * that should be isolated. The heap helps to accumulate all dynamic allocations in single memory + * region which is then added to modules memory domain. + */ +struct sys_heap *module_driver_heap_init(void); + +#endif + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @param alignment - Alignment in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * @note When CONFIG_USERSPACE not set function calls rballoc_align() + */ +void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, + uint32_t align); + +/** + * Allocates memory block from private module sys_heap if exists, otherwise call rmalloc. + * @param sys_heap - pointer to the sys_heap structure + * @param flags - Flags, see SOF_MEM_FLAG_... + * @param bytes - Size in bytes. + * @return - Pointer to the allocated memory or NULL if failed. + * + * * @note When CONFIG_USERSPACE not set function calls rmalloc() + */ +void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes); + +/** + * Similar to user_rmalloc(), guarantees that returned block is zeroed. + * + * @note When CONFIG_USERSPACE not set function calls rzalloc() + */ +void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes); + +/** + * Frees the memory block from private module sys_heap if exists. Otherwise call rfree. + * @param ptr Pointer to the memory block. + * + * @note User should take care to not free memory allocated from sys_heap + * with mod_drv_heap set to NULL. It will cause exception. + * + * When CONFIG_USERSPACE not set function calls rfree() + */ +void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem); + +/** + * Free private processing module heap. + * @param sys_heap pointer to the sys_heap structure. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + * Frees private module heap. + */ +void module_driver_heap_remove(struct sys_heap *mod_drv_heap); + +#endif /* __ZEPHYR_LIB_USERSPACE_HELPER_H__ */ diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c new file mode 100644 index 000000000000..74698d7712c1 --- /dev/null +++ b/zephyr/lib/userspace_helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2025 Intel Corporation. All rights reserved. +// +// Author: Jaroslaw Stelter +// Adrian Warecki + +/** + * \file + * \brief Zephyr userspace helper functions + * \authors Jaroslaw Stelter + * \authors Adrian Warecki + */ + +#include + +#include +#include + +#define MODULE_DRIVER_HEAP_CACHED CONFIG_SOF_ZEPHYR_HEAP_CACHED + +#if CONFIG_USERSPACE +struct sys_heap *module_driver_heap_init(void) +{ + struct sys_heap *mod_drv_heap = rballoc(SOF_MEM_FLAG_USER, sizeof(struct sys_heap)); + + if (!mod_drv_heap) + return NULL; + + void *mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, DRV_HEAP_SIZE, + CONFIG_MM_DRV_PAGE_SIZE); + if (!mem) { + rfree(mod_drv_heap); + return NULL; + } + + sys_heap_init(mod_drv_heap, mem, DRV_HEAP_SIZE); + mod_drv_heap->init_mem = mem; + mod_drv_heap->init_bytes = DRV_HEAP_SIZE; + + return mod_drv_heap; +} + +void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, + uint32_t align) +{ +#ifdef MODULE_DRIVER_HEAP_CACHED + const bool cached = (flags & SOF_MEM_FLAG_COHERENT) == 0; +#endif /* MODULE_DRIVER_HEAP_CACHED */ + + if (mod_drv_heap) { +#ifdef MODULE_DRIVER_HEAP_CACHED + if (cached) { + /* + * Zephyr sys_heap stores metadata at start of each + * heap allocation. To ensure no allocated cached buffer + * overlaps the same cacheline with the metadata chunk, + * align both allocation start and size of allocation + * to cacheline. As cached and non-cached allocations are + * mixed, same rules need to be followed for both type of + * allocations. + */ + align = MAX(PLATFORM_DCACHE_ALIGN, align); + bytes = ALIGN_UP(bytes, align); + } +#endif /* MODULE_DRIVER_HEAP_CACHED */ + void *mem = sys_heap_aligned_alloc(mod_drv_heap, align, bytes); +#ifdef MODULE_DRIVER_HEAP_CACHED + if (cached) + return sys_cache_cached_ptr_get(mem); +#endif /* MODULE_DRIVER_HEAP_CACHED */ + return mem; + } else { + return rballoc_align(flags, bytes, align); + } +} + +void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + if (mod_drv_heap) + return module_driver_heap_aligned_alloc(mod_drv_heap, flags, bytes, 0); + else + return rmalloc(flags, bytes); +} + +void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + void *ptr; + + ptr = module_driver_heap_rmalloc(mod_drv_heap, flags, bytes); + if (ptr) + memset(ptr, 0, bytes); + + return ptr; +} + +void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +{ + if (mod_drv_heap) { +#ifdef MODULE_DRIVER_HEAP_CACHED + if (is_cached(mem)) { + void *mem_uncached = sys_cache_uncached_ptr_get( + (__sparse_force void __sparse_cache *)mem); + + sys_cache_data_invd_range(mem, + sys_heap_usable_size(mod_drv_heap, mem_uncached)); + + mem = mem_uncached; + } +#endif + sys_heap_free(mod_drv_heap, mem); + } else { + rfree(mem); + } +} + +void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +{ + if (mod_drv_heap) { + rfree(mod_drv_heap->init_mem); + rfree(mod_drv_heap); + } +} + +#else /* CONFIG_USERSPACE */ + +void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + return rmalloc(flags, bytes); +} + +void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, + uint32_t align) +{ + return rballoc_align(flags, bytes, align); +} + +void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) +{ + return rzalloc(flags, bytes); +} + +void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +{ + rfree(mem); +} + +void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +{ } + +#endif /* CONFIG_USERSPACE */ From 2f924bda36ca68d3afe510c2b95083e62987f8f7 Mon Sep 17 00:00:00 2001 From: Jaroslaw Stelter Date: Mon, 4 Sep 2023 14:01:23 +0200 Subject: [PATCH 3/7] module_adapter: Allocate data buffers from MMU shared heap Allocates audio data buffer from shared memory heap. The security reqirements assume that the non-privileged modules data and code should be protected from other non-privileged modules access. The audio data could be accessible. This change is effective only when CONFIG_USERSPACE=y. Signed-off-by: Jaroslaw Stelter Signed-off-by: Adrian Warecki --- src/audio/module_adapter/module_adapter.c | 14 +++++++------- src/include/sof/audio/component.h | 8 ++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index e2a80f0fa9e3..1afd8c52354f 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -210,6 +210,7 @@ int module_adapter_prepare(struct comp_dev *dev) struct list_item *blist, *_blist; uint32_t buff_periods; uint32_t buff_size; /* size of local buffer */ + int memory_flags; int i = 0; comp_dbg(dev, "module_adapter_prepare() start"); @@ -342,11 +343,11 @@ int module_adapter_prepare(struct comp_dev *dev) module_adapter_check_data(mod, dev, sink); + memory_flags = user_get_buffer_memory_region(dev->drv); /* allocate memory for input buffers */ if (mod->max_sources) { mod->input_buffers = - rzalloc(SOF_MEM_FLAG_USER, - sizeof(*mod->input_buffers) * mod->max_sources); + rzalloc(memory_flags, sizeof(*mod->input_buffers) * mod->max_sources); if (!mod->input_buffers) { comp_err(dev, "failed to allocate input buffers"); return -ENOMEM; @@ -358,8 +359,7 @@ int module_adapter_prepare(struct comp_dev *dev) /* allocate memory for output buffers */ if (mod->max_sinks) { mod->output_buffers = - rzalloc(SOF_MEM_FLAG_USER, - sizeof(*mod->output_buffers) * mod->max_sinks); + rzalloc(memory_flags, sizeof(*mod->output_buffers) * mod->max_sinks); if (!mod->output_buffers) { comp_err(dev, "failed to allocate output buffers"); ret = -ENOMEM; @@ -425,7 +425,7 @@ int module_adapter_prepare(struct comp_dev *dev) list_for_item(blist, &dev->bsource_list) { size_t size = MAX(mod->deep_buff_bytes, mod->period_bytes); - mod->input_buffers[i].data = rballoc(SOF_MEM_FLAG_USER, size); + mod->input_buffers[i].data = rballoc(memory_flags, size); if (!mod->input_buffers[i].data) { comp_err(mod->dev, "Failed to alloc input buffer data"); ret = -ENOMEM; @@ -437,7 +437,7 @@ int module_adapter_prepare(struct comp_dev *dev) /* allocate memory for output buffer data */ i = 0; list_for_item(blist, &dev->bsink_list) { - mod->output_buffers[i].data = rballoc(SOF_MEM_FLAG_USER, md->mpd.out_buff_size); + mod->output_buffers[i].data = rballoc(memory_flags, md->mpd.out_buff_size); if (!mod->output_buffers[i].data) { comp_err(mod->dev, "Failed to alloc output buffer data"); ret = -ENOMEM; @@ -450,7 +450,7 @@ int module_adapter_prepare(struct comp_dev *dev) if (list_is_empty(&mod->raw_data_buffers_list)) { for (i = 0; i < mod->num_of_sinks; i++) { /* allocate not shared buffer */ - struct comp_buffer *buffer = buffer_alloc(buff_size, SOF_MEM_FLAG_USER, + struct comp_buffer *buffer = buffer_alloc(buff_size, memory_flags, PLATFORM_DCACHE_ALIGN, false); uint32_t flags; diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 40e43ef7c8a4..1440a775b400 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -1194,4 +1194,12 @@ void comp_init_performance_data(struct comp_dev *dev); */ bool comp_update_performance_data(struct comp_dev *dev, uint32_t cycles_used); +static inline int user_get_buffer_memory_region(const struct comp_driver *drv) +{ +#if CONFIG_USERSPACE + return drv->user_heap ? SOF_MEM_FLAG_USER_SHARED_BUFFER : SOF_MEM_FLAG_USER; +#else + return SOF_MEM_FLAG_USER; +#endif +} #endif /* __SOF_AUDIO_COMPONENT_H__ */ From 1a19ba97207e4ad07b5cd5121c0ad2767d7eadd3 Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Mon, 7 Jul 2025 18:50:03 +0200 Subject: [PATCH 4/7] audio: buffers: ring_buffer: Allocate memory from shared heap for userspace Allocates audio data buffer from shared memory heap. The security reqirements assume that the non-privileged modules data and code should be protected from other non-privileged modules access. The audio data could be accessible. This change is effective only when CONFIG_USERSPACE=y. Signed-off-by: Adrian Warecki --- src/audio/buffers/ring_buffer.c | 15 ++++++++------- src/include/sof/audio/ring_buffer.h | 5 ++++- src/ipc/ipc4/helper.c | 20 +++++++++----------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index b6cbc23060e2..a71a27022e0a 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -279,17 +280,16 @@ static const struct audio_buffer_ops audio_buffer_ops = { .reset = ring_buffer_reset }; -struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, bool is_shared, +struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_available, + size_t min_free_space, bool is_shared, uint32_t id) { struct ring_buffer *ring_buffer; + int memory_flags = (is_shared ? SOF_MEM_FLAG_COHERENT : 0) | + user_get_buffer_memory_region(dev->drv); /* allocate ring_buffer structure */ - if (is_shared) - ring_buffer = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ring_buffer)); - else - ring_buffer = rzalloc(SOF_MEM_FLAG_USER, sizeof(*ring_buffer)); + ring_buffer = rzalloc(memory_flags, sizeof(*ring_buffer)); if (!ring_buffer) return NULL; @@ -359,7 +359,8 @@ struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_spa ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *) - rballoc_align(SOF_MEM_FLAG_USER, ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + rballoc_align(memory_flags, ring_buffer->data_buffer_size, + PLATFORM_DCACHE_ALIGN); if (!ring_buffer->_data_buffer) goto err; diff --git a/src/include/sof/audio/ring_buffer.h b/src/include/sof/audio/ring_buffer.h index 222b102f2ae9..1ef712a56743 100644 --- a/src/include/sof/audio/ring_buffer.h +++ b/src/include/sof/audio/ring_buffer.h @@ -97,6 +97,7 @@ * always means "buffer full" */ +struct comp_dev; struct ring_buffer; struct sof_audio_stream_params; @@ -114,6 +115,7 @@ struct ring_buffer { /** * + * @param dev pointer to the DP module device structure * @param min_available minimum data available in queue required by the module using * ring_buffer's source api * @param min_free_space minimum buffer space in queue required by the module using @@ -122,7 +124,8 @@ struct ring_buffer { * @param id a stream ID, accessible later by sink_get_id/source_get_id * */ -struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, bool is_shared, +struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_available, + size_t min_free_space, bool is_shared, uint32_t id); #endif /* __SOF_RING_BUFFER_H__ */ diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 4e2f54db3553..511917ceb768 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -603,25 +603,23 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP || source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { - struct sof_source *source = audio_buffer_get_source(&buffer->audio_buffer); - struct sof_sink *sink = audio_buffer_get_sink(&buffer->audio_buffer); + bool dp_on_source = source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP; + struct sof_source *src = audio_buffer_get_source(&buffer->audio_buffer); + struct sof_sink *snk = audio_buffer_get_sink(&buffer->audio_buffer); - ring_buffer = ring_buffer_create(source_get_min_available(source), - sink_get_min_free_space(sink), + ring_buffer = ring_buffer_create(dp_on_source ? source : sink, + source_get_min_available(src), + sink_get_min_free_space(snk), audio_buffer_is_shared(&buffer->audio_buffer), buf_get_id(buffer)); if (!ring_buffer) goto free; - } - if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) /* data destination module needs to use ring_buffer */ - audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, false, /* at_input */ - &ring_buffer->audio_buffer); - else if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - /* data source module needs to use ring_buffer */ - audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, true, /* at_input */ + audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, dp_on_source, &ring_buffer->audio_buffer); + } + #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ /* * Connect and bind the buffer to both source and sink components with LL processing been From bd6abf05455cfa3584f2aaeba92e93e75b2b7dbf Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Mon, 17 Feb 2025 17:48:39 +0100 Subject: [PATCH 5/7] module_adapter: Separate generic_module_is_ready_to_process function Extract generic code from the module_is_ready_to_process function which can be used by other adapters, in particular the userspace module adapter. Signed-off-by: Adrian Warecki --- .../sof/audio/module_adapter/module/generic.h | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 1001c4ba450c..3a30b97172eb 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -163,6 +163,26 @@ int module_prepare(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks); +static inline +bool generic_module_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, + int num_of_sources, + struct sof_sink **sinks, + int num_of_sinks) +{ + int i; + + for (i = 0; i < num_of_sources; i++) + if (source_get_data_available(sources[i]) < source_get_min_available(sources[i])) + return false; + + for (i = 0; i < num_of_sinks; i++) + if (sink_get_free_size(sinks[i]) < sink_get_min_free_space(sinks[i])) + return false; + + return true; +} + static inline bool module_is_ready_to_process(struct processing_module *mod, struct sof_source **sources, @@ -170,7 +190,6 @@ bool module_is_ready_to_process(struct processing_module *mod, struct sof_sink **sinks, int num_of_sinks) { - int i; const struct module_interface *const ops = mod->dev->drv->adapter_ops; /* LL module has to be always ready for processing */ @@ -182,15 +201,8 @@ bool module_is_ready_to_process(struct processing_module *mod, /* default action - the module is ready if there's enough data for processing and enough * space to store result. IBS/OBS as declared in init_instance */ - for (i = 0; i < num_of_sources; i++) - if (source_get_data_available(sources[i]) < source_get_min_available(sources[i])) - return false; - - for (i = 0; i < num_of_sinks; i++) - if (sink_get_free_size(sinks[i]) < sink_get_min_free_space(sinks[i])) - return false; - - return true; + return generic_module_is_ready_to_process(mod, sources, num_of_sources, sinks, + num_of_sinks); } int module_process_sink_src(struct processing_module *mod, From 754ebd66549feca3f51c7c8cffd694a248c5cda7 Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Thu, 27 Feb 2025 17:41:18 +0100 Subject: [PATCH 6/7] lib_manager: Public lib_manager_get_instance_bss_address function Remove static keyword from lib_manager_get_instance_bss_address function and add function declaration. Signed-off-by: Adrian Warecki --- src/include/sof/lib_manager.h | 12 ++++++++++++ src/library_manager/lib_manager.c | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index f9b7f16c0b0c..ef32b5d75e87 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -186,6 +186,18 @@ int lib_manager_register_module(const uint32_t component_id); */ const struct sof_man_fw_desc *lib_manager_get_library_manifest(int module_id); +/* + * \brief Get address and size of the bss section for given module instance + * + * param[in] instance_id - instance id + * param[in] mod - module manifest + * param[out] va_addr - address of the bss section + * param[out] size - size of the bss section + */ +void lib_manager_get_instance_bss_address(uint32_t instance_id, + const struct sof_man_module *mod, + void __sparse_cache **va_addr, size_t *size); + /* * \brief Free module * diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index c153c61a308b..6170bbe1ccdc 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -273,9 +273,9 @@ static int lib_manager_unload_libcode_modules(const uint32_t module_id) } #endif /* CONFIG_LIBCODE_MODULE_SUPPORT */ -static void lib_manager_get_instance_bss_address(uint32_t instance_id, - const struct sof_man_module *mod, - void __sparse_cache **va_addr, size_t *size) +void lib_manager_get_instance_bss_address(uint32_t instance_id, + const struct sof_man_module *mod, + void __sparse_cache **va_addr, size_t *size) { *size = mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length / mod->instance_max_count * PAGE_SZ; From d3adf654dd53413ae7cfea68d7c990eccd74fa20 Mon Sep 17 00:00:00 2001 From: Jaroslaw Stelter Date: Fri, 3 Mar 2023 13:18:53 +0100 Subject: [PATCH 7/7] zephyr: userspace: Fix compilation options When CONFIG_USERSPACE enabled, the merging of global symbols ends with SOF failures. The merging must be disabled when CONFIG_USERSPACE is set. The patch fix also disabling -fno-inline-functions option which works differently for clang, completely disabling function inlining. Signed-off-by: Jaroslaw Stelter --- zephyr/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 6edc4246c63f..de22a83f574e 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -522,6 +522,10 @@ if (NOT CONFIG_COMPILER_INLINE_FUNCTION_OPTION) target_compile_options(SOF INTERFACE -fno-inline-functions) endif() +if (CONFIG_USERSPACE) +target_compile_options(SOF INTERFACE -mno-global-merge) +endif() + # SOF needs `typeof`, `__VA_ARGS__` and maybe other GNU C99 # extensions. TODO other flags required ? target_compile_options(SOF INTERFACE $<$: -std=gnu99>)