From 6bf097185692a9a3691c18f1a2ffe4052181e777 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 4 Apr 2025 10:10:52 +0200 Subject: [PATCH] llext: fix auxiliary module ordering Fully automated module dependency tree generation isn't planned for SOF yet, so for now we do this manually. At the moment the sorting is completely random, relying on Python's os.scandir() output. This patch makes it possible to order auxiliary LLEXT modules explicitly. It supports up to 9 dependency levels with level 1 being the most shallow one, i.e. only end-user modules can depend on it. If any level 1 modules depend on further modules, those are called "level 2" modules etc. For this llext targets in respective CMakeLists.txt files have to be called, e.g. "aux1_iir" - see existing examples. This then guarantees correct ordering and successful run-time symbol resolution. Signed-off-by: Guennadi Liakhovetski --- scripts/xtensa-build-zephyr.py | 137 ++++++++++++++++-------------- src/math/CMakeLists.txt | 8 +- src/math/fir_llext/CMakeLists.txt | 2 +- src/math/iir_llext/CMakeLists.txt | 2 +- 4 files changed, 77 insertions(+), 72 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index 0b9ce18b2fb0..5eaa19aa3648 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -40,6 +40,7 @@ import dataclasses import concurrent.futures as concurrent import tarfile +import re from west import configuration as west_config @@ -967,73 +968,77 @@ def install_lib(platform, sof_output_dir, abs_build_dir, platform_wconfig): sof_lib_dir.mkdir(parents=True, exist_ok=True) - with os.scandir(str(abs_build_dir)) as iter: - for entry in iter: - if (not entry.is_dir or - not entry.name.endswith('_llext')): - continue - - entry_path = pathlib.Path(entry.path) - - uuids = entry_path / 'llext.uuid' - if not os.path.exists(uuids): - print(f"Directory {entry.name} has no llext.uuid file. Skipping.") - continue + dirlist = [d for d in os.listdir(str(abs_build_dir)) if os.path.isdir(abs_build_dir / d)] + # Find all auxiliary modules, they're called "aux1_", "aux2_", etc., + # where lower numbers mean fewer dependent levels, so we need to sort + # them in reversed order to have all dependencies satisfied + pattern_aux = re.compile(r'^aux\d_.+_llext$') + aux = sorted([d for d in dirlist if pattern_aux.match(d)], reverse=True) + llext_dirs = [d for d in dirlist if d.endswith('_llext') and d not in aux] + final_list = aux + llext_dirs + + for entry in final_list: + entry_path = abs_build_dir / entry + + uuids = entry_path / 'llext.uuid' + if not os.path.exists(uuids): + print(f"Directory {entry} has no llext.uuid file. Skipping.") + continue - # replace '_llext' with '.llext', e.g. - # eq_iir_llext/eq_iir.llext - llext_base = entry.name[:-6] - llext_file = llext_base + '.llext' - lib_name = '' - - lib_fname = entry_path / 'lib_name.txt' - if os.path.exists(lib_fname): - with open(lib_fname, 'r') as libs_f: - lib_name = libs_f.read() - if lib_name not in libs.keys(): - libs[lib_name] = [] - libs[lib_name].append(str(entry_path / llext_file)) - else: - dst = sof_lib_dir / llext_file - - rimage_cfg = entry_path / 'rimage_config.toml' - llext_input = entry_path / (llext_base + '.llext') - llext_output = entry_path / (llext_file + '.ri') - - # See why the shlex() parsing step is required at - # https://docs.zephyrproject.org/latest/develop/west/sign.html#rimage - # and in Zephyr commit 030b740bd1ec - sign_cmd = [rimage_cmd, "-o", str(llext_output), - "-e", "-c", str(rimage_cfg), - "-k", str(signing_key), "-l", "-r"] - if _ws_args is not None: - sign_cmd.extend(shlex.split(_ws_args)) - sign_cmd.append(str(llext_input)) - execute_command(sign_cmd, cwd=west_top) - - # An intuitive way to make this multiline would be - # with (open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, - # open(llext_output.with_suffix('.llext.xman'), 'rb') as fman): - # but a Python version, used on Windows errored out on this. - # Thus we're left with a choice between a 150-character - # long line and an illogical split like this - with open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, open( - llext_output.with_suffix('.ri.xman'), 'rb') as fman: - # Concatenate the manifest and the llext - shutil.copyfileobj(fman, fdst) - shutil.copyfileobj(fllext, fdst) - - # Create symbolic links for all UUIDs - with open(uuids, 'r') as uuids_f: - for uuid in uuids_f: - if os.path.exists(lib_fname): - if lib_name not in lib_uuids.keys(): - lib_uuids[lib_name] = [] - lib_uuids[lib_name].append(uuid.strip()) - else: - linkname = uuid.strip() + '.bin' - symlink_or_copy(sof_lib_dir, llext_file, - sof_lib_dir, linkname) + # replace '_llext' with '.llext', e.g. + # eq_iir_llext/eq_iir.llext + llext_base = entry[:-6] + llext_file = llext_base + '.llext' + lib_name = '' + + lib_fname = entry_path / 'lib_name.txt' + if os.path.exists(lib_fname): + with open(lib_fname, 'r') as libs_f: + lib_name = libs_f.read() + if lib_name not in libs.keys(): + libs[lib_name] = [] + libs[lib_name].append(str(entry_path / llext_file)) + else: + dst = sof_lib_dir / llext_file + + rimage_cfg = entry_path / 'rimage_config.toml' + llext_input = entry_path / (llext_base + '.llext') + llext_output = entry_path / (llext_file + '.ri') + + # See why the shlex() parsing step is required at + # https://docs.zephyrproject.org/latest/develop/west/sign.html#rimage + # and in Zephyr commit 030b740bd1ec + sign_cmd = [rimage_cmd, "-o", str(llext_output), + "-e", "-c", str(rimage_cfg), + "-k", str(signing_key), "-l", "-r"] + if _ws_args is not None: + sign_cmd.extend(shlex.split(_ws_args)) + sign_cmd.append(str(llext_input)) + execute_command(sign_cmd, cwd=west_top) + + # An intuitive way to make this multiline would be + # with (open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, + # open(llext_output.with_suffix('.llext.xman'), 'rb') as fman): + # but a Python version, used on Windows errored out on this. + # Thus we're left with a choice between a 150-character + # long line and an illogical split like this + with open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, open( + llext_output.with_suffix('.ri.xman'), 'rb') as fman: + # Concatenate the manifest and the llext + shutil.copyfileobj(fman, fdst) + shutil.copyfileobj(fllext, fdst) + + # Create symbolic links for all UUIDs + with open(uuids, 'r') as uuids_f: + for uuid in uuids_f: + if os.path.exists(lib_fname): + if lib_name not in lib_uuids.keys(): + lib_uuids[lib_name] = [] + lib_uuids[lib_name].append(uuid.strip()) + else: + linkname = uuid.strip() + '.bin' + symlink_or_copy(sof_lib_dir, llext_file, + sof_lib_dir, linkname) lib_install_dir = sof_output_dir / platform if args.key_type_subdir != "none": diff --git a/src/math/CMakeLists.txt b/src/math/CMakeLists.txt index c635d9c917fd..fe8fa5d18ee9 100644 --- a/src/math/CMakeLists.txt +++ b/src/math/CMakeLists.txt @@ -44,8 +44,8 @@ if(CONFIG_BINARY_LOGARITHM_FIXED) endif() if(CONFIG_MATH_FIR STREQUAL "m") - add_subdirectory(fir_llext ${PROJECT_BINARY_DIR}/fir_llext) - add_dependencies(app fir) + add_subdirectory(fir_llext ${PROJECT_BINARY_DIR}/aux1_fir_llext) + add_dependencies(app aux1_fir) elseif(CONFIG_MATH_FIR) list(APPEND base_files fir_generic.c fir_hifi2ep.c fir_hifi3.c fir_hifi5.c) endif() @@ -55,8 +55,8 @@ if(CONFIG_MATH_FFT) endif() if(CONFIG_MATH_IIR STREQUAL "m") - add_subdirectory(iir_llext ${PROJECT_BINARY_DIR}/iir_llext) - add_dependencies(app iir) + add_subdirectory(iir_llext ${PROJECT_BINARY_DIR}/aux1_iir_llext) + add_dependencies(app aux1_iir) elseif(CONFIG_MATH_IIR) if(CONFIG_MATH_IIR_DF2T) list(APPEND base_files iir_df2t_generic.c iir_df2t_hifi3.c iir_df2t.c) diff --git a/src/math/fir_llext/CMakeLists.txt b/src/math/fir_llext/CMakeLists.txt index c3c0005be21b..688a07fa1ffb 100644 --- a/src/math/fir_llext/CMakeLists.txt +++ b/src/math/fir_llext/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) 2024 Intel Corporation. # SPDX-License-Identifier: Apache-2.0 -sof_llext_build("fir" +sof_llext_build("aux1_fir" SOURCES fir_common.c ../fir_generic.c ../fir_hifi2ep.c diff --git a/src/math/iir_llext/CMakeLists.txt b/src/math/iir_llext/CMakeLists.txt index 4b3afb6b05e6..f6f2a7c328fa 100644 --- a/src/math/iir_llext/CMakeLists.txt +++ b/src/math/iir_llext/CMakeLists.txt @@ -15,7 +15,7 @@ else() endif() if(CONFIG_MATH_IIR_DF1 OR CONFIG_MATH_IIR_DF2T) - sof_llext_build("iir" + sof_llext_build("aux1_iir" SOURCES iir.c ${df1} ${df2t} LIB openmodules )