diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index b4f53e7cf7f1..2b0032f8d2d5 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -24,6 +24,7 @@ #include #include #include +#include LOG_MODULE_REGISTER(buffer, CONFIG_SOF_LOG_LEVEL); @@ -235,6 +236,15 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align, tr_dbg(&buffer_tr, "buffer_alloc()"); +#if CONFIG_USERSPACE + /* + * cache control is not possible in user-space, so cross-core + * buffers must be allocated as coherent + */ + if (is_shared && k_is_user_context()) + flags |= SOF_MEM_FLAG_COHERENT; +#endif + /* validate request */ if (size == 0) { tr_err(&buffer_tr, "new size = %zu is invalid", size); @@ -273,6 +283,15 @@ struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_siz return NULL; } +#if CONFIG_USERSPACE + /* + * cache control is not possible in user-space, so cross-core + * buffers must be allocated as coherent + */ + if (is_shared && k_is_user_context()) + flags |= SOF_MEM_FLAG_COHERENT; +#endif + /* Align preferred size to a multiple of the minimum size */ if (preferred_size % minimum_size) preferred_size += minimum_size - preferred_size % minimum_size; diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index a71a27022e0a..0f931cfaeac4 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -13,6 +13,8 @@ #include #include +#include + LOG_MODULE_REGISTER(ring_buffer, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(ring_buffer); @@ -53,6 +55,14 @@ static inline void ring_buffer_invalidate_shared(struct ring_buffer *ring_buffer if (!ring_buffer_is_shared(ring_buffer)) return; +#if CONFIG_USERSPACE + /* user-space shared buffers are allocated as uncached */ + if (k_is_user_context()) { + __ASSERT_NO_MSG(sys_cache_is_ptr_cached(ptr) == false); + return; + } +#endif + /* wrap-around? */ if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) { /* writeback till the end of circular buffer */ @@ -72,6 +82,14 @@ static inline void ring_buffer_writeback_shared(struct ring_buffer *ring_buffer, if (!ring_buffer_is_shared(ring_buffer)) return; +#if CONFIG_USERSPACE + /* user-space shared buffers are allocated as uncached */ + if (k_is_user_context()) { + __ASSERT_NO_MSG(sys_cache_is_ptr_cached(ptr) == false); + return; + } +#endif + /* wrap-around? */ if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) { /* writeback till the end of circular buffer */ @@ -355,6 +373,15 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl */ ring_buffer->data_buffer_size = 3 * max_ibs_obs; +#if CONFIG_USERSPACE + /* + * cache control is not possible in user-space, so cross-core + * buffers must be allocated as coherent + */ + if (is_shared && k_is_user_context()) + memory_flags |= SOF_MEM_FLAG_COHERENT; +#endif + /* allocate data buffer - always in cached memory alias */ ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); diff --git a/src/include/sof/audio/audio_stream.h b/src/include/sof/audio/audio_stream.h index 9703ec9cadea..ebe58022c209 100644 --- a/src/include/sof/audio/audio_stream.h +++ b/src/include/sof/audio/audio_stream.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -733,6 +734,14 @@ static inline void audio_stream_invalidate(struct audio_stream *buffer, uint32_t tail_size = bytes - head_size; } +#if CONFIG_USERSPACE + /* user-space shared buffers are allocated as uncached */ + if (k_is_user_context()) { + __ASSERT_NO_MSG(sys_cache_is_ptr_cached(buffer->addr) == false); + return; + } +#endif + dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->r_ptr, head_size); if (tail_size) dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->addr, @@ -756,6 +765,14 @@ static inline void audio_stream_writeback(struct audio_stream *buffer, uint32_t tail_size = bytes - head_size; } +#if CONFIG_USERSPACE + /* user-space shared buffers are allocated as uncached */ + if (k_is_user_context()) { + __ASSERT_NO_MSG(sys_cache_is_ptr_cached(buffer->addr) == false); + return; + } +#endif + dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, head_size); if (tail_size) dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->addr, diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index c8b98255ed4f..72a27387b53d 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -257,20 +258,32 @@ bool buffer_params_match(struct comp_buffer *buffer, static inline void buffer_stream_invalidate(struct comp_buffer *buffer, uint32_t bytes) { #if CONFIG_INCOHERENT - if (audio_buffer_is_shared(&buffer->audio_buffer)) + if (audio_buffer_is_shared(&buffer->audio_buffer)) { +#if CONFIG_USERSPACE + /* user-space shared buffers are allocated as uncached */ + if (k_is_user_context()) + return; +#endif audio_stream_invalidate(&buffer->stream, bytes); + } #endif } static inline void buffer_stream_writeback(struct comp_buffer *buffer, uint32_t bytes) { #if CONFIG_INCOHERENT - if (audio_buffer_is_shared(&buffer->audio_buffer)) + if (audio_buffer_is_shared(&buffer->audio_buffer)) { +#if CONFIG_USERSPACE + /* user-space shared buffers are allocated as uncached */ + if (k_is_user_context()) + return; +#endif + audio_stream_writeback(&buffer->stream, bytes); + } #endif } - /* * Attach a new buffer at the beginning of the list. Note, that "head" must * really be the head of the list, not a list head within another buffer. We