diff --git a/posix/include/rtos/alloc.h b/posix/include/rtos/alloc.h index 195c1d64258f..6b690ecbc895 100644 --- a/posix/include/rtos/alloc.h +++ b/posix/include/rtos/alloc.h @@ -38,22 +38,24 @@ * the first two positions are reserved for SOF_BUF_ flags */ - /** \brief Indicates we should return DMA-able memory. */ + /** \brief Allocate DMA-able memory. */ #define SOF_MEM_FLAG_DMA BIT(2) -/** \brief Indicates that original content should not be copied by realloc. */ +/** \brief realloc() skips copying the original content. */ #define SOF_MEM_FLAG_NO_COPY BIT(3) -/** \brief Indicates that if we should return uncached address. */ +/** \brief Allocate uncached address. */ #define SOF_MEM_FLAG_COHERENT BIT(4) -/** \brief Indicates that if we should return L3 address. */ +/** \brief Allocate L3 address. */ #define SOF_MEM_FLAG_L3 BIT(5) -/** \brief Indicates that if we should return Low power memory address. */ +/** \brief Allocate Low power memory address. */ #define SOF_MEM_FLAG_LOW_POWER BIT(6) -/** \brief Indicates that if we should return kernel memory address. */ +/** \brief Allocate kernel memory address. */ #define SOF_MEM_FLAG_KERNEL BIT(7) -/** \brief Indicates that if we should return user memory address. */ +/** \brief Allocate user memory address. */ #define SOF_MEM_FLAG_USER BIT(8) -/** \brief Indicates that if we should return shared user memory address. */ +/** \brief Allocate shared user memory address. */ #define SOF_MEM_FLAG_USER_SHARED_BUFFER BIT(9) +/** \brief Use allocation method for large buffers. */ +#define SOF_MEM_FLAG_LARGE_BUFFER BIT(10) /** @} */ diff --git a/posix/include/rtos/userspace_helper.h b/posix/include/rtos/userspace_helper.h index dda440a33ac4..09a52b7804d5 100644 --- a/posix/include/rtos/userspace_helper.h +++ b/posix/include/rtos/userspace_helper.h @@ -26,7 +26,7 @@ struct sys_heap; /** * Initialize private processing module heap. * @param N/A. - * @return pointer to the sys_heap structure. + * @return pointer to the k_heap structure. * * @note * Function used only when CONFIG_USERSPACE is set. @@ -34,69 +34,12 @@ struct sys_heap; * that should be isolated. The heap helps to accumulate all dynamic allocations in single memory * region which is then added to modules memory domain. */ -static inline struct sys_heap *module_driver_heap_init(void) +static inline struct k_heap *module_driver_heap_init(void) { return NULL; } - #endif -/** - * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). - * @param sys_heap - pointer to the sys_heap structure - * @param flags - Flags, see SOF_MEM_FLAG_... - * @param bytes - Size in bytes. - * @param alignment - Alignment in bytes. - * @return Pointer to the allocated memory or NULL if failed. - * - * @note When CONFIG_USERSPACE not set function calls rballoc_align() - */ -static inline void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, - size_t bytes, uint32_t align) -{ - return rballoc_align(flags, bytes, align); -} - -/** - * Allocates memory block from private module sys_heap if exists, otherwise call rmalloc. - * @param sys_heap - pointer to the sys_heap structure - * @param flags - Flags, see SOF_MEM_FLAG_... - * @param bytes - Size in bytes. - * @return - Pointer to the allocated memory or NULL if failed. - * - * * @note When CONFIG_USERSPACE not set function calls rmalloc() - */ -static inline void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, - size_t bytes) -{ - return rmalloc(flags, bytes); -} - -/** - * Similar to user_rmalloc(), guarantees that returned block is zeroed. - * - * @note When CONFIG_USERSPACE not set function calls rzalloc() - */ -static inline void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, - size_t bytes) -{ - return rzalloc(flags, bytes); -} - -/** - * Frees the memory block from private module sys_heap if exists. Otherwise call rfree. - * @param ptr Pointer to the memory block. - * - * @note User should take care to not free memory allocated from sys_heap - * with module_driver_heap set to NULL. It will cause exception. - * - * When CONFIG_USERSPACE not set function calls rfree() - */ -static inline void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) -{ - rfree(mem); -} - /** * Free private processing module heap. * @param sys_heap pointer to the sys_heap structure. @@ -105,7 +48,7 @@ static inline void module_driver_heap_free(struct sys_heap *mod_drv_heap, void * * Function used only when CONFIG_USERSPACE is set. * Frees private module heap. */ -static inline void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +static inline void module_driver_heap_remove(struct k_heap *mod_drv_heap) { } #endif /* __RTOS_USERSPACE_HELPER_H__ */ diff --git a/src/audio/module_adapter/library/userspace_proxy.c b/src/audio/module_adapter/library/userspace_proxy.c index 3bc6124cebc4..d6b1516dfb3c 100644 --- a/src/audio/module_adapter/library/userspace_proxy.c +++ b/src/audio/module_adapter/library/userspace_proxy.c @@ -49,10 +49,11 @@ static int userspace_proxy_memory_init(struct userspace_context *user, { /* Add module private heap to memory partitions */ struct k_mem_partition heap_part = { .attr = K_MEM_PARTITION_P_RW_U_RW }; + struct sys_heap *heap = &drv->user_heap->heap; k_mem_region_align(&heap_part.start, &heap_part.size, - POINTER_TO_UINT(drv->user_heap->init_mem), - drv->user_heap->init_bytes, CONFIG_MM_DRV_PAGE_SIZE); + POINTER_TO_UINT(heap->init_mem), + heap->init_bytes, CONFIG_MM_DRV_PAGE_SIZE); tr_dbg(&userspace_proxy_tr, "Heap partition %p + %zx, attr = %u", UINT_TO_POINTER(heap_part.start), heap_part.size, heap_part.attr); @@ -63,8 +64,8 @@ static int userspace_proxy_memory_init(struct userspace_context *user, struct k_mem_partition heap_cached_part = { .attr = K_MEM_PARTITION_P_RW_U_RW }; k_mem_region_align(&heap_cached_part.start, &heap_cached_part.size, - POINTER_TO_UINT(sys_cache_cached_ptr_get(drv->user_heap->init_mem)), - drv->user_heap->init_bytes, CONFIG_MM_DRV_PAGE_SIZE); + POINTER_TO_UINT(sys_cache_cached_ptr_get(heap->init_mem)), + heap->init_bytes, CONFIG_MM_DRV_PAGE_SIZE); tr_dbg(&userspace_proxy_tr, "Cached heap partition %p + %zx, attr = %u", UINT_TO_POINTER(heap_cached_part.start), heap_cached_part.size, @@ -135,7 +136,7 @@ int userspace_proxy_create(struct userspace_context **user_ctx, const struct com tr_dbg(&userspace_proxy_tr, "userspace create"); - user = sys_heap_alloc(drv->user_heap, sizeof(struct userspace_context)); + user = k_heap_alloc(drv->user_heap, sizeof(struct userspace_context), K_FOREVER); if (!user) return -ENOMEM; @@ -184,7 +185,7 @@ int userspace_proxy_create(struct userspace_context **user_ctx, const struct com error_dom: rfree(domain); error: - sys_heap_free(drv->user_heap, user); + k_heap_free(drv->user_heap, user); return ret; } @@ -192,7 +193,7 @@ void userspace_proxy_destroy(const struct comp_driver *drv, struct userspace_con { tr_dbg(&userspace_proxy_tr, "userspace proxy destroy"); rfree(user_ctx->comp_dom); - sys_heap_free(drv->user_heap, user_ctx); + k_heap_free(drv->user_heap, user_ctx); } /** diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 0094a138a243..1f0862d2a590 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -137,10 +137,11 @@ struct container_chunk { static struct module_resource *container_get(struct processing_module *mod) { struct module_resources *res = &mod->priv.resources; + struct k_heap *mod_heap = res->heap; struct module_resource *container; if (list_is_empty(&res->free_cont_list)) { - struct container_chunk *chunk = rzalloc(SOF_MEM_FLAG_USER, sizeof(*chunk)); + struct container_chunk *chunk = sof_heap_alloc(mod_heap, 0, sizeof(*chunk), 0); int i; if (!chunk) { @@ -148,6 +149,8 @@ static struct module_resource *container_get(struct processing_module *mod) return NULL; } + memset(chunk, 0, sizeof(*chunk)); + list_item_append(&chunk->chunk_list, &res->cont_chunk_list); for (i = 0; i < ARRAY_SIZE(chunk->containers); i++) list_item_append(&chunk->containers[i].list, &res->free_cont_list); @@ -180,7 +183,6 @@ void *mod_balloc_align(struct processing_module *mod, size_t size, size_t alignm { struct module_resources *res = &mod->priv.resources; struct module_resource *container; - void *ptr; MEM_API_CHECK_THREAD(res); @@ -195,7 +197,8 @@ void *mod_balloc_align(struct processing_module *mod, size_t size, size_t alignm } /* Allocate buffer memory for module */ - ptr = rballoc_align(SOF_MEM_FLAG_USER, size, alignment); + void *ptr = sof_heap_alloc(res->heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER, + size, alignment); if (!ptr) { comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", @@ -231,7 +234,6 @@ void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, { struct module_resources *res = &mod->priv.resources; struct module_resource *container; - void *ptr; MEM_API_CHECK_THREAD(res); @@ -246,7 +248,7 @@ void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, } /* Allocate memory for module */ - ptr = rmalloc_align(flags, size, alignment); + void *ptr = sof_heap_alloc(res->heap, flags, size, alignment); if (!ptr) { comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", @@ -276,8 +278,7 @@ EXPORT_SYMBOL(mod_alloc_ext); * Like comp_data_blob_handler_new() but the handler is automatically freed. */ #if CONFIG_COMP_BLOB -struct comp_data_blob_handler * -mod_data_blob_handler_new(struct processing_module *mod) +struct comp_data_blob_handler *mod_data_blob_handler_new(struct processing_module *mod) { struct module_resources *res = &mod->priv.resources; struct comp_data_blob_handler *bhp; @@ -347,7 +348,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * switch (container->type) { case MOD_RES_HEAP: - rfree(container->ptr); + sof_heap_free(res->heap, container->ptr); res->heap_usage -= container->size; return 0; #if CONFIG_COMP_BLOB @@ -588,6 +589,7 @@ int module_reset(struct processing_module *mod) void mod_free_all(struct processing_module *mod) { struct module_resources *res = &mod->priv.resources; + struct k_heap *mod_heap = res->heap; struct list_item *list; struct list_item *_list; @@ -611,7 +613,7 @@ void mod_free_all(struct processing_module *mod) container_of(list, struct container_chunk, chunk_list); list_item_del(&chunk->chunk_list); - rfree(chunk); + sof_heap_free(mod_heap, chunk); } /* Make sure resource lists and accounting are reset */ diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 05f7bc705599..cf36da4e1034 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -44,33 +45,85 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, return module_adapter_new_ext(drv, config, spec, NULL, NULL); } -static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, - const struct comp_ipc_config *config) +#if CONFIG_MM_DRV +#define PAGE_SZ CONFIG_MM_DRV_PAGE_SIZE +#else +#include +#define PAGE_SZ HOST_PAGE_SIZE +#endif + +static struct k_heap *module_adapter_dp_heap_new(const struct comp_ipc_config *config) { - struct comp_dev *dev = comp_alloc(drv, sizeof(*dev)); + /* src-lite with 8 channels has been seen allocating 14k in one go */ + /* FIXME: the size will be derived from configuration */ + const size_t heap_size = 20 * 1024; - if (!dev) { - comp_cl_err(drv, "failed to allocate memory for comp_dev"); + /* Keep uncached to match the default SOF heap! */ + uint8_t *mod_heap_mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + heap_size, PAGE_SZ); + + if (!mod_heap_mem) return NULL; - } - /* allocate module information. - * for DP shared modules this struct must be accessible from all cores - * Unfortunately at this point there's no information of components the module - * will be bound to. So we need to allocate shared memory for each DP module - * To be removed when pipeline 2.0 is ready + struct k_heap *mod_heap = (struct k_heap *)mod_heap_mem; + const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap), 8); + void *mod_heap_buf = mod_heap_mem + heap_prefix_size; + + k_heap_init(mod_heap, mod_heap_buf, heap_size - heap_prefix_size); + + return mod_heap; +} + +static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, + const struct comp_ipc_config *config) +{ + struct k_heap *mod_heap; + /* + * For DP shared modules the struct processing_module object must be + * accessible from all cores. Unfortunately at this point there's no + * information of components the module will be bound to. So we need to + * allocate shared memory for each DP module. + * To be removed when pipeline 2.0 is ready. */ - int flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; + uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; + + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_USERSPACE) && + !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { + mod_heap = module_adapter_dp_heap_new(config); + if (!mod_heap) { + comp_cl_err(drv, "Failed to allocate DP module heap"); + return NULL; + } + } else { + mod_heap = drv->user_heap; + } - struct processing_module *mod = module_driver_heap_rzalloc(drv->user_heap, flags, - sizeof(*mod)); + struct processing_module *mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0); if (!mod) { - comp_err(dev, "failed to allocate memory for module"); + comp_cl_err(drv, "failed to allocate memory for module"); + goto emod; + } + + memset(mod, 0, sizeof(*mod)); + mod->priv.resources.heap = mod_heap; + + /* + * Would be difficult to optimize the allocation to use cache. Only if + * the whole currently active topology is running on the primary core, + * then it can be cached. Effectively it can be only cached in + * single-core configurations. + */ + struct comp_dev *dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); + + if (!dev) { + comp_cl_err(drv, "failed to allocate memory for comp_dev"); goto err; } + memset(dev, 0, sizeof(*dev)); + comp_init(drv, dev, sizeof(*dev)); dev->ipc_config = *config; mod->dev = dev; dev->mod = mod; @@ -78,20 +131,23 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv return mod; err: - module_driver_heap_free(drv->user_heap, dev); + sof_heap_free(mod_heap, mod); +emod: + if (mod_heap != drv->user_heap) + rfree(mod_heap); return NULL; } static void module_adapter_mem_free(struct processing_module *mod) { - const struct comp_driver *drv = mod->dev->drv; + struct k_heap *mod_heap = mod->priv.resources.heap; #if CONFIG_IPC_MAJOR_4 - module_driver_heap_free(drv->user_heap, mod->priv.cfg.input_pins); + sof_heap_free(mod_heap, mod->priv.cfg.input_pins); #endif - module_driver_heap_free(drv->user_heap, mod->dev); - module_driver_heap_free(drv->user_heap, mod); + sof_heap_free(mod_heap, mod->dev); + sof_heap_free(mod_heap, mod); } /* @@ -572,11 +628,9 @@ int module_adapter_params(struct comp_dev *dev, struct sof_ipc_stream_params *pa #endif /* allocate stream_params each time */ - if (mod->stream_params) - rfree(mod->stream_params); + mod_free(mod, mod->stream_params); - mod->stream_params = rzalloc(SOF_MEM_FLAG_USER, - sizeof(*mod->stream_params) + params->ext_data_length); + mod->stream_params = mod_alloc(mod, sizeof(*mod->stream_params) + params->ext_data_length); if (!mod->stream_params) return -ENOMEM; @@ -1278,7 +1332,7 @@ int module_adapter_reset(struct comp_dev *dev) buffer_zero(buffer); } - rfree(mod->stream_params); + mod_free(mod, mod->stream_params); mod->stream_params = NULL; comp_dbg(dev, "done"); @@ -1310,9 +1364,9 @@ void module_adapter_free(struct comp_dev *dev) buffer_free(buffer); } + mod_free(mod, mod->stream_params); mod_free_all(mod); - rfree(mod->stream_params); module_adapter_mem_free(mod); } EXPORT_SYMBOL(module_adapter_free); diff --git a/src/audio/module_adapter/module_adapter_ipc4.c b/src/audio/module_adapter/module_adapter_ipc4.c index 8e1d13661dec..54ee3dd94f5e 100644 --- a/src/audio/module_adapter/module_adapter_ipc4.c +++ b/src/audio/module_adapter/module_adapter_ipc4.c @@ -136,9 +136,9 @@ int module_adapter_init_data(struct comp_dev *dev, if (cfgsz == (sizeof(*cfg) + pinsz)) { dst->nb_input_pins = n_in; dst->nb_output_pins = n_out; - dst->input_pins = module_driver_heap_rmalloc(dev->drv->user_heap, - SOF_MEM_FLAG_USER | - SOF_MEM_FLAG_COHERENT, pinsz); + dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.heap, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + pinsz, 0); if (!dst->input_pins) return -ENOMEM; diff --git a/src/idc/idc.c b/src/idc/idc.c index 466f84eee63f..c3656008f1c5 100644 --- a/src/idc/idc.c +++ b/src/idc/idc.c @@ -184,20 +184,21 @@ static int idc_prepare(uint32_t comp_id) /* we're running LL on different core, so allocate our own task */ if (!dev->task && dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) { /* allocate task for shared component */ - dev->task = module_driver_heap_rzalloc(dev->drv->user_heap, SOF_MEM_FLAG_USER, - sizeof(*dev->task)); + dev->task = sof_heap_alloc(dev->drv->user_heap, SOF_MEM_FLAG_USER, + sizeof(*dev->task), 0); if (!dev->task) { ret = -ENOMEM; goto out; } + memset(dev->task, 0, sizeof(*dev->task)); ret = schedule_task_init_ll(dev->task, SOF_UUID(idc_task_uuid), SOF_SCHEDULE_LL_TIMER, dev->priority, comp_task, dev, dev->ipc_config.core, 0); if (ret < 0) { - module_driver_heap_free(dev->drv->user_heap, dev->task); + sof_heap_free(dev->drv->user_heap, dev->task); goto out; } } diff --git a/src/idc/zephyr_idc.c b/src/idc/zephyr_idc.c index afd30628a5d3..f1af9d53a85b 100644 --- a/src/idc/zephyr_idc.c +++ b/src/idc/zephyr_idc.c @@ -185,6 +185,11 @@ void idc_init_thread(void) k_p4wq_enable_static_thread(q_zephyr_idc + cpu, _p4threads_q_zephyr_idc + cpu, BIT(cpu)); + /* + * Assign SOF system heap to the IDC thread. Otherwise by default it + * uses the Zephyr heap for DP stack allocation + */ + k_thread_heap_assign(_p4threads_q_zephyr_idc + cpu, sof_sys_heap_get()); } #endif /* CONFIG_MULTICORE */ diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 02639f823419..dfc840d67bb9 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -591,7 +591,7 @@ struct comp_driver { * Intended to replace the ops field. * Currently used by module_adapter. */ - struct sys_heap *user_heap; /**< Userspace heap */ + struct k_heap *user_heap; /**< Userspace heap */ }; /** \brief Holds constant pointer to component driver */ @@ -879,13 +879,14 @@ static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, size_t * Use uncached address everywhere to access components to rule out * multi-core failures. TODO: verify if cached alias may be used in some cases */ - struct comp_dev *dev = module_driver_heap_rzalloc(drv->user_heap, - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - bytes); + struct comp_dev *dev = sof_heap_alloc(drv->user_heap, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + bytes, 0); if (!dev) return NULL; + memset(dev, 0, sizeof(*dev)); comp_init(drv, dev, bytes); return dev; diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 3762fcde7791..b5ce2075b2f0 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -50,7 +50,7 @@ static inline void comp_free(struct comp_dev *dev) if ((dev->is_shared || dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) && dev->task) { schedule_task_free(dev->task); - module_driver_heap_free(dev->drv->user_heap, dev->task); + sof_heap_free(dev->drv->user_heap, dev->task); dev->task = NULL; } diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 484c0d6856f7..535227336b9e 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -132,6 +132,7 @@ struct module_resources { struct list_item cont_chunk_list; /**< Memory container chunks */ size_t heap_usage; size_t heap_high_water_mark; + struct k_heap *heap; #if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) k_tid_t rsrc_mngr; #endif diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 8bdc8545c808..cde00a4c15a3 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -736,7 +736,7 @@ int lib_manager_register_module(const uint32_t component_id) const struct sof_man_module *mod = lib_manager_get_module_manifest(component_id); const struct sof_uuid *uid = (struct sof_uuid *)&mod->uuid; struct comp_driver_info *new_drv_info; - struct sys_heap *drv_heap = NULL; + struct k_heap *drv_heap = NULL; struct comp_driver *drv = NULL; int ret = -ENOMEM; @@ -765,12 +765,14 @@ int lib_manager_register_module(const uint32_t component_id) } #endif /* CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP */ - drv = module_driver_heap_rmalloc(drv_heap, SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(struct comp_driver)); + drv = sof_heap_alloc(drv_heap, SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(struct comp_driver), 0); if (!drv) { tr_err(&lib_manager_tr, "failed to allocate comp_driver"); goto cleanup; } + + memset(drv, 0, sizeof(*drv)); drv->user_heap = drv_heap; lib_manager_prepare_module_adapter(drv, uid); @@ -785,7 +787,7 @@ int lib_manager_register_module(const uint32_t component_id) cleanup: if (ret < 0) { rfree(new_drv_info); - module_driver_heap_free(drv_heap, drv); + sof_heap_free(drv_heap, drv); module_driver_heap_remove(drv_heap); } diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 991d9a30eb49..6181a34a1a4e 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -578,7 +578,7 @@ int scheduler_dp_task_init(struct task **task, uint32_t options) { void __sparse_cache *p_stack = NULL; - struct sys_heap *const user_heap = mod->dev->drv->user_heap; + struct k_heap *const user_heap = mod->dev->drv->user_heap; /* memory allocation helper structure */ struct { @@ -598,13 +598,14 @@ int scheduler_dp_task_init(struct task **task, * As the structure contains zephyr kernel specific data, it must be located in * shared, non cached memory */ - task_memory = module_driver_heap_rzalloc(user_heap, SOF_MEM_FLAG_USER | - SOF_MEM_FLAG_COHERENT, sizeof(*task_memory)); + task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); if (!task_memory) { tr_err(&dp_tr, "memory alloc failed"); return -ENOMEM; } + memset(task_memory, 0, sizeof(*task_memory)); /* allocate stack - must be aligned and cached so a separate alloc */ p_stack = user_stack_allocate(stack_size, options); if (!p_stack) { @@ -666,7 +667,7 @@ int scheduler_dp_task_init(struct task **task, /* k_object_free looks for a pointer in the list, any invalid value can be passed */ k_object_free(task_memory->pdata.sem); k_object_free(task_memory->pdata.thread); - module_driver_heap_free(user_heap, task_memory); + sof_heap_free(user_heap, task_memory); return ret; } diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 854eb211e904..447bf1754860 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -38,8 +38,16 @@ config SOF_ZEPHYR_HEAP_SIZE NOTE: Keep in mind that the heap size should not be greater than the physical memory size of the system defined in DT (and this includes baseFW text/data). +config SOF_USERSPACE_USE_SHARED_HEAP + bool "Use shared heap for SOF userspace modules" + depends on USERSPACE + help + When set a shared heap will be used for audio buffers between SOF + kernel and userspace modules. + config SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE hex "Size of the shared buffer heap for SOF userspace modules" + default 0x0 if !SOF_USERSPACE_USE_SHARED_HEAP default 0x1E000 if SOC_INTEL_ACE15_MTPM || SOC_INTEL_ACE20_LNL default 0x1A000 if SOC_INTEL_ACE30 default 0x0 @@ -87,6 +95,7 @@ config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE config SOF_USERSPACE_PROXY bool "Use userspace proxy to support userspace modules" select SOF_USERSPACE_USE_DRIVER_HEAP + select SOF_USERSPACE_USE_SHARED_HEAP help When set, userspace modules are launched inside a container created by userspace proxy. It is responsible for forwarding module function calls coming from sof running in diff --git a/zephyr/edf_schedule.c b/zephyr/edf_schedule.c index 8aca007137e7..cc224be4bb05 100644 --- a/zephyr/edf_schedule.c +++ b/zephyr/edf_schedule.c @@ -5,6 +5,7 @@ // Author: Bartosz Kokoszko #include +#include #include #include #include @@ -111,6 +112,7 @@ int scheduler_init_edf(void) k_thread_suspend(thread); + k_thread_heap_assign(thread, sof_sys_heap_get()); k_thread_cpu_mask_clear(thread); k_thread_cpu_mask_enable(thread, PLATFORM_PRIMARY_CORE_ID); k_thread_name_set(thread, "edf_workq"); diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index fe984808b682..116789ea6ead 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -25,26 +25,28 @@ * the first two positions are reserved for SOF_BUF_ flags */ -/** \name Heap zone flags +/** \name Allocation flags * @{ */ - /** \brief Indicates we should return DMA-able memory. */ + /** \brief Allocate DMA-able memory. */ #define SOF_MEM_FLAG_DMA BIT(2) -/** \brief Indicates that original content should not be copied by realloc. */ +/** \brief realloc() skips copying the original content. */ #define SOF_MEM_FLAG_NO_COPY BIT(3) -/** \brief Indicates that if we should return uncached address. */ +/** \brief Allocate uncached address. */ #define SOF_MEM_FLAG_COHERENT BIT(4) -/** \brief Indicates that if we should return L3 address. */ +/** \brief Allocate L3 address. */ #define SOF_MEM_FLAG_L3 BIT(5) -/** \brief Indicates that if we should return Low power memory address. */ +/** \brief Allocate Low power memory address. */ #define SOF_MEM_FLAG_LOW_POWER BIT(6) -/** \brief Indicates that if we should return kernel memory address. */ +/** \brief Allocate kernel memory address. */ #define SOF_MEM_FLAG_KERNEL BIT(7) -/** \brief Indicates that if we should return user memory address. */ +/** \brief Allocate user memory address. */ #define SOF_MEM_FLAG_USER BIT(8) -/** \brief Indicates that if we should return shared user memory address. */ +/** \brief Allocate shared user memory address. */ #define SOF_MEM_FLAG_USER_SHARED_BUFFER BIT(9) +/** \brief Use allocation method for large buffers. */ +#define SOF_MEM_FLAG_LARGE_BUFFER BIT(10) /** @} */ @@ -123,13 +125,14 @@ void l3_heap_save(void); void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, size_t alignment); void sof_heap_free(struct k_heap *heap, void *addr); +struct k_heap *sof_sys_heap_get(void); /* TODO: remove - debug only - only needed for linking */ static inline void heap_trace_all(int force) {} /** @}*/ -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP /** * Returns the start address of shared memory heap for buffers. * diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h index 23b69598745b..a09280077a64 100644 --- a/zephyr/include/rtos/userspace_helper.h +++ b/zephyr/include/rtos/userspace_helper.h @@ -30,7 +30,7 @@ struct userspace_context; /** * Initialize private processing module heap. * @param N/A. - * @return pointer to the sys_heap structure. + * @return pointer to the k_heap structure. * * @note * Function used only when CONFIG_USERSPACE is set. @@ -38,7 +38,7 @@ struct userspace_context; * that should be isolated. The heap helps to accumulate all dynamic allocations in single memory * region which is then added to modules memory domain. */ -struct sys_heap *module_driver_heap_init(void); +struct k_heap *module_driver_heap_init(void); /** * Add DP scheduler created thread to module memory domain. @@ -81,48 +81,6 @@ void *user_stack_allocate(size_t stack_size, uint32_t options); */ int user_stack_free(void *p_stack); -/** - * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). - * @param sys_heap - pointer to the sys_heap structure - * @param flags - Flags, see SOF_MEM_FLAG_... - * @param bytes - Size in bytes. - * @param alignment - Alignment in bytes. - * @return Pointer to the allocated memory or NULL if failed. - * - * @note When CONFIG_USERSPACE not set function calls rballoc_align() - */ -void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, - uint32_t align); - -/** - * Allocates memory block from private module sys_heap if exists, otherwise call rmalloc. - * @param sys_heap - pointer to the sys_heap structure - * @param flags - Flags, see SOF_MEM_FLAG_... - * @param bytes - Size in bytes. - * @return - Pointer to the allocated memory or NULL if failed. - * - * * @note When CONFIG_USERSPACE not set function calls rmalloc() - */ -void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes); - -/** - * Similar to user_rmalloc(), guarantees that returned block is zeroed. - * - * @note When CONFIG_USERSPACE not set function calls rzalloc() - */ -void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes); - -/** - * Frees the memory block from private module sys_heap if exists. Otherwise call rfree. - * @param ptr Pointer to the memory block. - * - * @note User should take care to not free memory allocated from sys_heap - * with mod_drv_heap set to NULL. It will cause exception. - * - * When CONFIG_USERSPACE not set function calls rfree() - */ -void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem); - /** * Free private processing module heap. * @param sys_heap pointer to the sys_heap structure. @@ -131,6 +89,6 @@ void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem); * Function used only when CONFIG_USERSPACE is set. * Frees private module heap. */ -void module_driver_heap_remove(struct sys_heap *mod_drv_heap); +void module_driver_heap_remove(struct k_heap *mod_drv_heap); #endif /* __ZEPHYR_LIB_USERSPACE_HELPER_H__ */ diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 255f5bfdcf98..946a07b9124a 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -105,8 +105,10 @@ static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE]; #undef SHARED_BUFFER_HEAP_MEM_SIZE #define SHARED_BUFFER_HEAP_MEM_SIZE ROUND_UP(CONFIG_SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE, \ HOST_PAGE_SIZE) +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP __section(".shared_heap_mem") static uint8_t __aligned(HOST_PAGE_SIZE) shared_heapmem[SHARED_BUFFER_HEAP_MEM_SIZE]; +#endif #endif /* CONFIG_USERSPACE */ __section(".heap_mem") static uint8_t __aligned(HOST_PAGE_SIZE) heapmem[HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE]; @@ -149,7 +151,7 @@ extern char _end[], _heap_sentry[]; static struct k_heap sof_heap; -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP static struct k_heap shared_buffer_heap; static bool is_shared_buffer_heap_pointer(void *ptr) @@ -181,7 +183,7 @@ size_t get_shared_buffer_heap_size(void) { return ROUND_DOWN(SHARED_BUFFER_HEAP_MEM_SIZE, HOST_PAGE_SIZE); } -#endif /* CONFIG_USERSPACE */ +#endif /* CONFIG_SOF_USERSPACE_USE_SHARED_HEAP */ #if CONFIG_L3_HEAP static struct k_heap l3_heap; @@ -375,6 +377,11 @@ SYS_INIT(virtual_heap_init, POST_KERNEL, 1); #endif /* CONFIG_VIRTUAL_HEAP */ +struct k_heap *sof_sys_heap_get(void) +{ + return &sof_heap; +} + static void *heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes) { k_spinlock_key_t key; @@ -466,7 +473,7 @@ void *rmalloc_align(uint32_t flags, size_t bytes, uint32_t alignment) #else k_panic(); #endif -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP } else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) { heap = &shared_buffer_heap; #endif @@ -541,8 +548,7 @@ EXPORT_SYMBOL(rzalloc); * @param align Alignment in bytes. * @return Pointer to the allocated memory or NULL if failed. */ -void *rballoc_align(uint32_t flags, size_t bytes, - uint32_t align) +void *rballoc_align(uint32_t flags, size_t bytes, uint32_t align) { struct k_heap *heap; @@ -555,15 +561,15 @@ void *rballoc_align(uint32_t flags, size_t bytes, tr_err(&zephyr_tr, "L3_HEAP not available."); return NULL; #endif -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP } else if (flags & SOF_MEM_FLAG_USER_SHARED_BUFFER) { heap = &shared_buffer_heap; #endif /* CONFIG_USERSPACE */ } else { #if CONFIG_VIRTUAL_HEAP - /* Use virtual heap if it is available */ - if (virtual_buffers_heap) - return virtual_heap_alloc(virtual_buffers_heap, flags, bytes, align); + /* Use virtual heap if it is available */ + if (virtual_buffers_heap) + return virtual_heap_alloc(virtual_buffers_heap, flags, bytes, align); #endif /* CONFIG_VIRTUAL_HEAP */ heap = &sof_heap; @@ -598,7 +604,7 @@ void rfree(void *ptr) } #endif -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP if (is_shared_buffer_heap_pointer(ptr)) { heap_free(&shared_buffer_heap, ptr); return; @@ -616,6 +622,9 @@ EXPORT_SYMBOL(rfree); void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, size_t alignment) { + if (flags & SOF_MEM_FLAG_LARGE_BUFFER) + return rballoc_align(flags, bytes, alignment); + if (!heap) heap = &sof_heap; @@ -627,17 +636,17 @@ void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, void sof_heap_free(struct k_heap *heap, void *addr) { - if (!heap) - heap = &sof_heap; - - heap_free(heap, addr); + if (heap && addr) + heap_free(heap, addr); + else + rfree(addr); } static int heap_init(void) { sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE - SHARED_BUFFER_HEAP_MEM_SIZE); -#if CONFIG_USERSPACE +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP sys_heap_init(&shared_buffer_heap.heap, shared_heapmem, SHARED_BUFFER_HEAP_MEM_SIZE); #endif diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index 41c374b1df43..6e46bdbf75ce 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -29,9 +29,9 @@ K_APPMEM_PARTITION_DEFINE(common_partition); -struct sys_heap *module_driver_heap_init(void) +struct k_heap *module_driver_heap_init(void) { - struct sys_heap *mod_drv_heap = rballoc(SOF_MEM_FLAG_USER, sizeof(struct sys_heap)); + struct k_heap *mod_drv_heap = rballoc(SOF_MEM_FLAG_USER, sizeof(*mod_drv_heap)); if (!mod_drv_heap) return NULL; @@ -43,90 +43,17 @@ struct sys_heap *module_driver_heap_init(void) return NULL; } - sys_heap_init(mod_drv_heap, mem, USER_MOD_HEAP_SIZE); - mod_drv_heap->init_mem = mem; - mod_drv_heap->init_bytes = USER_MOD_HEAP_SIZE; + k_heap_init(mod_drv_heap, mem, USER_MOD_HEAP_SIZE); + mod_drv_heap->heap.init_mem = mem; + mod_drv_heap->heap.init_bytes = USER_MOD_HEAP_SIZE; return mod_drv_heap; } -void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, - uint32_t align) -{ -#ifdef MODULE_DRIVER_HEAP_CACHED - const bool cached = (flags & SOF_MEM_FLAG_COHERENT) == 0; -#endif /* MODULE_DRIVER_HEAP_CACHED */ - - if (mod_drv_heap) { -#ifdef MODULE_DRIVER_HEAP_CACHED - if (cached) { - /* - * Zephyr sys_heap stores metadata at start of each - * heap allocation. To ensure no allocated cached buffer - * overlaps the same cacheline with the metadata chunk, - * align both allocation start and size of allocation - * to cacheline. As cached and non-cached allocations are - * mixed, same rules need to be followed for both type of - * allocations. - */ - align = MAX(PLATFORM_DCACHE_ALIGN, align); - bytes = ALIGN_UP(bytes, align); - } -#endif /* MODULE_DRIVER_HEAP_CACHED */ - void *mem = sys_heap_aligned_alloc(mod_drv_heap, align, bytes); -#ifdef MODULE_DRIVER_HEAP_CACHED - if (cached) - return sys_cache_cached_ptr_get(mem); -#endif /* MODULE_DRIVER_HEAP_CACHED */ - return mem; - } else { - return rballoc_align(flags, bytes, align); - } -} - -void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) -{ - if (mod_drv_heap) - return module_driver_heap_aligned_alloc(mod_drv_heap, flags, bytes, 0); - else - return rmalloc(flags, bytes); -} - -void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) -{ - void *ptr; - - ptr = module_driver_heap_rmalloc(mod_drv_heap, flags, bytes); - if (ptr) - memset(ptr, 0, bytes); - - return ptr; -} - -void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) +void module_driver_heap_remove(struct k_heap *mod_drv_heap) { if (mod_drv_heap) { -#ifdef MODULE_DRIVER_HEAP_CACHED - if (is_cached(mem)) { - void *mem_uncached = sys_cache_uncached_ptr_get( - (__sparse_force void __sparse_cache *)mem); - - sys_cache_data_invd_range(mem, - sys_heap_usable_size(mod_drv_heap, mem_uncached)); - - mem = mem_uncached; - } -#endif - sys_heap_free(mod_drv_heap, mem); - } else { - rfree(mem); - } -} - -void module_driver_heap_remove(struct sys_heap *mod_drv_heap) -{ - if (mod_drv_heap) { - rfree(mod_drv_heap->init_mem); + rfree(mod_drv_heap->heap.init_mem); rfree(mod_drv_heap); } } @@ -174,28 +101,7 @@ int user_stack_free(void *p_stack) return 0; } -void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) -{ - return rmalloc(flags, bytes); -} - -void *module_driver_heap_aligned_alloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes, - uint32_t align) -{ - return rballoc_align(flags, bytes, align); -} - -void *module_driver_heap_rzalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) -{ - return rzalloc(flags, bytes); -} - -void module_driver_heap_free(struct sys_heap *mod_drv_heap, void *mem) -{ - rfree(mem); -} - -void module_driver_heap_remove(struct sys_heap *mod_drv_heap) +void module_driver_heap_remove(struct k_heap *mod_drv_heap) { } #endif /* CONFIG_USERSPACE */