dp: userspace: Modify dp scheduler to work with USERSPACE

K_USER option flag is ignored until CONFIG_USERSPACE is not set.
When set, it creates thread in non_priviledged mode.
Without proper support such thread will fail.
This change is required to run DP scheduler with CONFIG_USERSPACE
flag set.

When DP scheduler supports non-privileged modules use of irq_lock is not
allowed. Therefore for this configuration it uses static semaphores one
per core. These could be accesses from user space thread.

New created thread with K_USER option (non-privileged) must be added to
module memory domain to get access to its private memory space.
Additionally allocate scheduler data and task_memory from shared pool.

Signed-off-by: Jaroslaw Stelter <Jaroslaw.Stelter@intel.com>
Signed-off-by: Adrian Warecki <adrian.warecki@intel.com>

Author: jxstelter

src/audio/pipeline/pipeline-schedule.c

@@ -26,6 +26,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <sof/audio/module_adapter/module/generic.h>
 
 LOG_MODULE_DECLARE(pipe, CONFIG_SOF_LOG_LEVEL);
 
@@ -39,7 +40,7 @@ SOF_DEFINE_REG_UUID(dp_task);
  * current static stack size for each DP component
  * TODO: to be taken from module manifest
  */
-#define TASK_DP_STACK_SIZE 8192
+#define TASK_DP_STACK_SIZE CONFIG_SOF_STACK_SIZE
 
 #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */
 
@@ -397,7 +398,8 @@ int pipeline_comp_dp_task_init(struct comp_dev *comp)
 					     &ops,
 					     mod,
 					     comp->ipc_config.core,
-					     TASK_DP_STACK_SIZE);
+					     TASK_DP_STACK_SIZE,
+					     mod->user_ctx ? K_USER : 0);
 		if (ret < 0)
 			return ret;
 	}

src/include/sof/schedule/dp_schedule.h

@@ -67,13 +67,15 @@ int scheduler_dp_init(void);
  * \param[in] mod pointer to the module to be run
  * \param[in] core CPU the thread should run on
  * \param[in] stack_size size of stack for a zephyr task
+ * \param[in] options task options used for creation
  */
 int scheduler_dp_task_init(struct task **task,
 			   const struct sof_uuid_entry *uid,
 			   const struct task_ops *ops,
 			   struct processing_module *mod,
 			   uint16_t core,
-			   size_t stack_size);
+			   size_t stack_size,
+			   uint32_t options);
 
 /**
  * \brief Extract information about scheduler's tasks

src/schedule/zephyr_dp_schedule.c

@@ -8,6 +8,7 @@
 #include <sof/audio/component.h>
 #include <sof/audio/module_adapter/module/generic.h>
 #include <rtos/task.h>
+#include <rtos/userspace_helper.h>
 #include <stdint.h>
 #include <sof/schedule/dp_schedule.h>
 #include <sof/schedule/ll_schedule.h>
@@ -17,6 +18,8 @@
 #include <rtos/interrupt.h>
 #include <zephyr/kernel.h>
 #include <zephyr/sys_clock.h>
+#include <zephyr/sys/sem.h>
+#include <zephyr/sys/mutex.h>
 #include <sof/lib/notifier.h>
 #include <ipc4/base_fw.h>
 
@@ -34,20 +37,58 @@ struct scheduler_dp_data {
 
 struct task_dp_pdata {
 	k_tid_t thread_id;		/* zephyr thread ID */
-	struct k_thread thread;		/* memory space for a thread */
+	struct k_thread *thread;	/* pointer to the kernels' thread object */
+	struct k_thread thread_struct;	/* thread object for kernel threads */
 	uint32_t deadline_clock_ticks;	/* dp module deadline in Zephyr ticks */
 	k_thread_stack_t __sparse_cache *p_stack;	/* pointer to thread stack */
 	size_t stack_size;		/* size of the stack in bytes */
-	struct k_sem sem;		/* semaphore for task scheduling */
+	struct k_sem *sem;		/* pointer to semaphore for task scheduling */
+	struct k_sem sem_struct;	/* semaphore for task scheduling for kernel threads */
 	struct processing_module *mod;	/* the module to be scheduled */
 	uint32_t ll_cycles_to_start;    /* current number of LL cycles till delayed start */
 };
 
+#ifdef CONFIG_USERSPACE
+/* Single CPU-wide lock
+ * The irq_lock is not available for USERSPACE (non-privileged) threads.
+ * Therefore semaphore is used to control critical section.
+ */
+#define DP_LOCK_INIT(i, _)	Z_SEM_INITIALIZER(dp_lock[i], 1, 1)
+#define DP_LOCK_INIT_LIST	LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,))
+
+/* User threads don't need access to this array. Access is performed from
+ * the kernel space via a syscall. Array must be placed in special section
+ * to be qualified as initialized by the gen_kobject_list.py script.
+ */
+static
+STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST };
+
+static inline unsigned int scheduler_dp_lock(uint16_t core)
+{
+	k_sem_take(&dp_lock[core], K_FOREVER);
+	return core;
+}
+
+static inline void scheduler_dp_unlock(unsigned int key)
+{
+	k_sem_give(&dp_lock[key]);
+}
+static inline void scheduler_dp_grant(k_tid_t thread_id, uint16_t core)
+{
+	k_thread_access_grant(thread_id, &dp_lock[core]);
+}
+
+#else /* CONFIG_USERSPACE */
+
+static inline void scheduler_dp_grant(k_tid_t thread_id, uint16_t core)
+{
+}
+
 /* Single CPU-wide lock
  * as each per-core instance if dp-scheduler has separate structures, it is enough to
  * use irq_lock instead of cross-core spinlocks
  */
-static inline unsigned int scheduler_dp_lock(void)
+static inline unsigned int scheduler_dp_lock(uint16_t core)
 {
 	return irq_lock();
 }
@@ -56,6 +97,7 @@ static inline void scheduler_dp_unlock(unsigned int key)
 {
 	irq_unlock(key);
 }
+#endif
 
 /* dummy LL task - to start LL on secondary cores */
 static enum task_state scheduler_dp_ll_tick_dummy(void *data)
@@ -226,7 +268,7 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *
 	unsigned int lock_key;
 	struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP);
 
-	lock_key = scheduler_dp_lock();
+	lock_key = scheduler_dp_lock(cpu_get_id());
 	list_for_item(tlist, &dp_sch->tasks) {
 		curr_task = container_of(tlist, struct task, list);
 		pdata = curr_task->priv_data;
@@ -256,7 +298,7 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *
 
 				/* trigger the task */
 				curr_task->state = SOF_TASK_STATE_RUNNING;
-				k_sem_give(&pdata->sem);
+				k_sem_give(pdata->sem);
 			}
 		}
 	}
@@ -271,7 +313,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task)
 
 
 	/* this is asyn cancel - mark the task as canceled and remove it from scheduling */
-	lock_key = scheduler_dp_lock();
+	lock_key = scheduler_dp_lock(cpu_get_id());
 
 	task->state = SOF_TASK_STATE_CANCEL;
 	list_item_del(&task->list);
@@ -281,7 +323,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task)
 		schedule_task_cancel(&dp_sch->ll_tick_src);
 
 	/* if the task is waiting on a semaphore - let it run and self-terminate */
-	k_sem_give(&pdata->sem);
+	k_sem_give(pdata->sem);
 	scheduler_dp_unlock(lock_key);
 
 	/* wait till the task has finished, if there was any task created */
@@ -294,6 +336,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task)
 static int scheduler_dp_task_free(void *data, struct task *task)
 {
 	struct task_dp_pdata *pdata = task->priv_data;
+	int ret;
 
 	scheduler_dp_task_cancel(data, task);
 
@@ -305,12 +348,19 @@ static int scheduler_dp_task_free(void *data, struct task *task)
 		pdata->thread_id = NULL;
 	}
 
+#ifdef CONFIG_USERSPACE
+	if (pdata->sem != &pdata->sem_struct)
+		k_object_free(pdata->sem);
+	if (pdata->thread != &pdata->thread_struct)
+		k_object_free(pdata->thread);
+#endif
+
 	/* free task stack */
-	rfree((__sparse_force void *)pdata->p_stack);
+	ret = user_stack_free((__sparse_force void *)pdata->p_stack);
 	pdata->p_stack = NULL;
 
 	/* all other memory has been allocated as a single malloc, will be freed later by caller */
-	return 0;
+	return ret;
 }
 
 /* Thread function called in component context, on target core */
@@ -329,14 +379,14 @@ static void dp_thread_fn(void *p1, void *p2, void *p3)
 		 * the thread is started immediately after creation, it will stop on semaphore
 		 * Semaphore will be released once the task is ready to process
 		 */
-		k_sem_take(&task_pdata->sem, K_FOREVER);
+		k_sem_take(task_pdata->sem, K_FOREVER);
 
 		if (task->state == SOF_TASK_STATE_RUNNING)
 			state = task_run(task);
 		else
 			state = task->state;	/* to avoid undefined variable warning */
 
-		lock_key = scheduler_dp_lock();
+		lock_key = scheduler_dp_lock(task->core);
 		/*
 		 * check if task is still running, may have been canceled by external call
 		 * if not, set the state returned by run procedure
@@ -382,7 +432,7 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta
 	uint64_t deadline_clock_ticks;
 	int ret;
 
-	lock_key = scheduler_dp_lock();
+	lock_key = scheduler_dp_lock(cpu_get_id());
 
 	if (task->state != SOF_TASK_STATE_INIT &&
 	    task->state != SOF_TASK_STATE_CANCEL &&
@@ -392,19 +442,36 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta
 	}
 
 	/* create a zephyr thread for the task */
-	pdata->thread_id = k_thread_create(&pdata->thread, (__sparse_force void *)pdata->p_stack,
+	pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)pdata->p_stack,
 					   pdata->stack_size, dp_thread_fn, task, NULL, NULL,
-					   CONFIG_DP_THREAD_PRIORITY, K_USER, K_FOREVER);
+					   CONFIG_DP_THREAD_PRIORITY, task->flags, K_FOREVER);
+	if (!pdata->thread_id) {
+		tr_err(&dp_tr, "DP thread creation failed");
+		scheduler_dp_unlock(lock_key);
+		return -ECHILD;
+	}
 
+	k_thread_access_grant(pdata->thread_id, pdata->sem);
+	scheduler_dp_grant(pdata->thread_id, cpu_get_id());
 	/* pin the thread to specific core */
 	ret = k_thread_cpu_pin(pdata->thread_id, task->core);
 	if (ret < 0) {
-		tr_err(&dp_tr, "zephyr_dp_task_init(): zephyr task pin to core failed");
+		tr_err(&dp_tr, "zephyr task pin to core failed");
 		goto err;
 	}
 
-	/* start the thread,  it should immediately stop at a semaphore, so clean it */
-	k_sem_reset(&pdata->sem);
+#ifdef CONFIG_USERSPACE
+	if (task->flags & K_USER) {
+		ret = user_memory_init_shared(pdata->thread_id, pdata->mod);
+		if (ret < 0) {
+			tr_err(&dp_tr, "user_memory_init_shared() failed");
+			goto err;
+		}
+	}
+#endif /* CONFIG_USERSPACE */
+
+	/* start the thread, it should immediately stop at a semaphore, so clean it */
+	k_sem_init(pdata->sem, 0, 1);
 	k_thread_start(pdata->thread_id);
 
 	/* if there's no DP tasks scheduled yet, run ll tick source task */
@@ -474,7 +541,8 @@ int scheduler_dp_task_init(struct task **task,
 			   const struct task_ops *ops,
 			   struct processing_module *mod,
 			   uint16_t core,
-			   size_t stack_size)
+			   size_t stack_size,
+			   uint32_t options)
 {
 	void __sparse_cache *p_stack = NULL;
 
@@ -496,17 +564,15 @@ int scheduler_dp_task_init(struct task **task,
 	 * As the structure contains zephyr kernel specific data, it must be located in
 	 * shared, non cached memory
 	 */
-	task_memory = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT,
-			      sizeof(*task_memory));
+	task_memory = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT |
+			      SOF_MEM_FLAG_USER_SHARED, sizeof(*task_memory));
 	if (!task_memory) {
 		tr_err(&dp_tr, "zephyr_dp_task_init(): memory alloc failed");
 		return -ENOMEM;
 	}
 
 	/* allocate stack - must be aligned and cached so a separate alloc */
-	stack_size = Z_KERNEL_STACK_SIZE_ADJUST(stack_size);
-	p_stack = (__sparse_force void __sparse_cache *)
-		rballoc_align(SOF_MEM_FLAG_KERNEL, stack_size, Z_KERNEL_STACK_OBJ_ALIGN);
+	p_stack = user_stack_allocate(stack_size, options);
 	if (!p_stack) {
 		tr_err(&dp_tr, "zephyr_dp_task_init(): stack alloc failed");
 		ret = -ENOMEM;
@@ -515,33 +581,57 @@ int scheduler_dp_task_init(struct task **task,
 
 	/* internal SOF task init */
 	ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run,
-				 mod, core, 0);
+				 mod, core, options);
 	if (ret < 0) {
 		tr_err(&dp_tr, "zephyr_dp_task_init(): schedule_task_init failed");
 		goto err;
 	}
 
+	/* Point to ksem semaphore for kernel threads synchronization */
+	/* It will be overwritten for K_USER threads to dynamic ones.  */
+	task_memory->pdata.sem = &task_memory->pdata.sem_struct;
+	task_memory->pdata.thread = &task_memory->pdata.thread_struct;
+
+#ifdef CONFIG_USERSPACE
+	if (options & K_USER) {
+		task_memory->pdata.sem = k_object_alloc(K_OBJ_SEM);
+		if (!task_memory->pdata.sem) {
+			tr_err(&dp_tr, "Semaphore object allocation failed");
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		task_memory->pdata.thread = k_object_alloc(K_OBJ_THREAD);
+		if (!task_memory->pdata.thread) {
+			tr_err(&dp_tr, "Thread object allocation failed");
+			ret = -ENOMEM;
+			goto err;
+		}
+	}
+#endif /* CONFIG_USERSPACE */
+
 	/* initialize other task structures */
 	task_memory->task.ops.complete = ops->complete;
 	task_memory->task.ops.get_deadline = ops->get_deadline;
 	task_memory->task.state = SOF_TASK_STATE_INIT;
 	task_memory->task.core = core;
 
-	/* initialize semaprhore */
-	k_sem_init(&task_memory->pdata.sem, 0, 1);
-
 	/* success, fill the structures */
 	task_memory->task.priv_data = &task_memory->pdata;
 	task_memory->pdata.p_stack = p_stack;
 	task_memory->pdata.stack_size = stack_size;
 	task_memory->pdata.mod = mod;
 	*task = &task_memory->task;
 
-
 	return 0;
 err:
 	/* cleanup - free all allocated resources */
-	rfree((__sparse_force void *)p_stack);
+	if (user_stack_free((__sparse_force void *)p_stack))
+		tr_err(&dp_tr, "user_stack_free failed!");
+
+	/* k_object_free looks for a pointer in the list, any invalid value can be passed */
+	k_object_free(task_memory->pdata.sem);
+	k_object_free(task_memory->pdata.thread);
 	rfree(task_memory);
 	return ret;
 }
@@ -554,7 +644,7 @@ void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_
 	struct scheduler_dp_data *dp_sch =
 		(struct scheduler_dp_data *)scheduler_get_data(SOF_SCHEDULE_DP);
 
-	lock_key = scheduler_dp_lock();
+	lock_key = scheduler_dp_lock(cpu_get_id());
 	scheduler_get_task_info(scheduler_props, data_off_size,  &dp_sch->tasks);
 	scheduler_dp_unlock(lock_key);
 }