From d32591e77d4eb74d3499fb25c1fab436e56e91b5 Mon Sep 17 00:00:00 2001 From: Garrick Meeker Date: Tue, 17 Jun 2025 16:03:53 -0700 Subject: [PATCH 1/4] Initial version of install-universal command --- .../commands/install-universal/README.md | 59 +++++ .../cmd_install_universal.py | 248 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 extensions/commands/install-universal/README.md create mode 100644 extensions/commands/install-universal/cmd_install_universal.py diff --git a/extensions/commands/install-universal/README.md b/extensions/commands/install-universal/README.md new file mode 100644 index 0000000..a511d8e --- /dev/null +++ b/extensions/commands/install-universal/README.md @@ -0,0 +1,59 @@ +## [Install macOS/iOS universal binaries](cmd_install_universal.py) + +Install dependent packages as universal binaries packages. +Conan v2 introduces support for multiple architectures, e.g. -s arch=armv8|x86_64 +however this is largely limited to CMake. + +For CMake-based project that only depends on other CMake-based recipes, it's now +possible to run: + +``` +conan install . -pr:h universal -pr:b default -b missing +conan build . -pr:h universal -pr:b default +``` + +However, many other recipes use autotools or other build systems that don't support +universal binaries. This command skips the usual build() / package() steps and +runs lipo when a universal package is needed. + +**This command is in an experimental stage, feedback is welcome.** + +**Parameters** +* supports all arguments used by `conan install`, see `conan install_universal --help` + +Note: many arguments are not correctly passed to the single architecture installs. +Currently this includes -o and -s settings. The Conan dependency graph does not support +building multiple architectures so this executes Conan again in a new process. + +**Profile** +The multi-architecture must be specified in the host profile. +The build profile will likely be a single architecture, although the default +binary compatibility plugin does not know that universal binaries can be used +for single architecture profiles. + +``` +[settings] +arch=armv8|x86_64 +build_type=Release +compiler=apple-clang +compiler.cppstd=17 +compiler.libcxx=libc++ +compiler.version=15 +os=Macos +os.version=11.0 +``` + +Also make sure to use += not = if you define CXXFLAGS or you will override the -arch flag +to autotools. + +``` +[buildenv] +CFLAGS+=-fvisibility=hidden +CXXFLAGS+=-fvisibility=hidden -fvisibility-inlines-hidden +``` + +Usage: +``` +conan install-universal . -pr:h universal -pr:b default -b missing +conan build . -pr:h universal -pr:b default +``` diff --git a/extensions/commands/install-universal/cmd_install_universal.py b/extensions/commands/install-universal/cmd_install_universal.py new file mode 100644 index 0000000..949b1e7 --- /dev/null +++ b/extensions/commands/install-universal/cmd_install_universal.py @@ -0,0 +1,248 @@ +import json +import os +import shutil +from subprocess import run +import sys +import tempfile + +from conan.api.conan_api import ConanAPI +from conan.api.output import ConanOutput +from conan.cli import make_abs_path +from conan.cli.args import common_graph_args, validate_common_graph_args +from conan.cli.command import conan_command +from conan.cli.formatters.graph import format_graph_json +from conan.cli.printers import print_profiles +from conan.cli.printers.graph import print_graph_packages, print_graph_basic + + +@conan_command(group="Consumer", formatters={"json": format_graph_json}) +def install_universal(conan_api: ConanAPI, parser, *args): + """ + Install universal packages from the requirements specified in a recipe (conanfile.py or conanfile.txt). + + It can also be used to install packages without a conanfile, using the + --requires and --tool-requires arguments. + + If any requirement is not found in the local cache, it will iterate the remotes + looking for it. When the full dependency graph is computed, and all dependencies + recipes have been found, it will look for binary packages matching the current settings. + If no binary package is found for some or several dependencies, it will error, + unless the '--build' argument is used to build it from source. + + After installation of packages, the generators and deployers will be called. + """ + common_graph_args(parser) + parser.add_argument("-g", "--generator", action="append", help='Generators to use') + parser.add_argument("-of", "--output-folder", + help='The root output folder for generated and build files') + parser.add_argument("-d", "--deployer", action="append", + help="Deploy using the provided deployer to the output folder. " + "Built-in deployers: 'full_deploy', 'direct_deploy', 'runtime_deploy'") + parser.add_argument("--deployer-folder", + help="Deployer output folder, base build folder by default if not set") + parser.add_argument("--deployer-package", action="append", + help="Execute the deploy() method of the packages matching " + "the provided patterns") + parser.add_argument("--build-require", action='store_true', default=False, + help='Whether the provided path is a build-require') + parser.add_argument("--envs-generation", default=None, choices=["false"], + help="Generation strategy for virtual environment files for the root") + args = parser.parse_args(*args) + validate_common_graph_args(args) + # basic paths + cwd = os.getcwd() + path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None + source_folder = os.path.dirname(path) if args.path else cwd + output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None + + # Basic collaborators: remotes, lockfile, profiles + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] + overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None + lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, + partial=args.lockfile_partial, overrides=overrides) + profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) + print_profiles(profile_host, profile_build) + + # Graph computation (without installation of binaries) + gapi = conan_api.graph + if path: + deps_graph = gapi.load_graph_consumer(path, args.name, args.version, args.user, args.channel, + profile_host, profile_build, lockfile, remotes, + args.update, is_build_require=args.build_require) + else: + deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, profile_host, + profile_build, lockfile, remotes, args.update) + + # Handle universal packages + if profile_host.settings['os'] in ['Macos', 'iOS', 'watchOS', 'tvOS', 'visionOS'] and '|' in profile_host.settings['arch']: + for node in deps_graph.ordered_iterate(): + make_universal_conanfile(node.conanfile, args) + print(node.conanfile.name, node.conanfile.generate, node.conanfile.build) + + print_graph_basic(deps_graph) + deps_graph.report_graph_error() + gapi.analyze_binaries(deps_graph, args.build, remotes, update=args.update, lockfile=lockfile) + print_graph_packages(deps_graph) + + # Installation of binaries and consumer generators + conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) + ConanOutput().title("Finalizing install (deploy, generators)") + conan_api.install.install_consumer(deps_graph, args.generator, source_folder, output_folder, + deploy=args.deployer, deploy_package=args.deployer_package, + deploy_folder=args.deployer_folder, + envs_generation=args.envs_generation) + ConanOutput().success("Install finished successfully") + + # Update lockfile if necessary + lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, + clean=args.lockfile_clean) + conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) + return {"graph": deps_graph, + "conan_api": conan_api} + + +def make_universal_conanfile(conanfile, args): + def _generate(conanfile): + print('generate', conanfile.name) + pass + def _build(conanfile): + print('build', conanfile.name) + archs = str(conanfile.settings.arch).split('|') + ref = conanfile.ref + for arch in archs: + more_args = '' + if args.profile_build: + more_args += f' -pr:b {args.profile_build[0]}' + if args.profile_host: + more_args += f' -pr:h {args.profile_host[0]}' + for arch in archs: + ConanOutput().success(f"Building universal {ref} {arch}") + arch_folder = os.path.join(conanfile.build_folder, arch) + run(f'conan install --requires={ref}{more_args} -d direct_deploy --deployer-folder "{arch_folder}" -s arch={arch} -b missing', shell=True) + def _package(conanfile): + print('package', conanfile.name) + archs = str(conanfile.settings.arch).split('|') + lipo_tree(conanfile.package_folder, [os.path.join(conanfile.build_folder, arch, 'direct_deploy', conanfile.name) for arch in archs]) + if conanfile.settings.get_safe('arch', '') and conanfile.package_type not in ('header-library', 'build-scripts', 'python-require'): + setattr(conanfile, 'generate', _generate.__get__(conanfile, type(conanfile))) + setattr(conanfile, 'build', _build.__get__(conanfile, type(conanfile))) + setattr(conanfile, 'package', _package.__get__(conanfile, type(conanfile))) + + +# Lipo support + +# These are for optimization only, to avoid unnecessarily reading files. +_binary_exts = ['.a', '.dylib'] +_regular_exts = [ + '.h', '.hpp', '.hxx', '.c', '.cc', '.cxx', '.cpp', '.m', '.mm', '.txt', '.md', '.html', '.jpg', '.png', '.class' +] + + +def is_macho_binary(filename): + ext = os.path.splitext(filename)[1] + if ext in _binary_exts: + return True + if ext in _regular_exts: + return False + with open(filename, "rb") as f: + header = f.read(4) + if header == b'\xcf\xfa\xed\xfe': + # cffaedfe is Mach-O binary + return True + elif header == b'\xca\xfe\xba\xbe': + # cafebabe is Mach-O fat binary + return True + elif header == b'!\n': + # ar archive + return True + return False + + +def is_macho_fat_binary(filename): + ext = os.path.splitext(filename)[1] + if ext in _binary_exts: + return True + if ext in _regular_exts: + return False + with open(filename, "rb") as f: + header = f.read(4) + if header == b'\xcf\xfa\xed\xfe': + # cffaedfe is Mach-O binary + return False + elif header == b'\xca\xfe\xba\xbe': + # cafebabe is Mach-O fat binary + return True + elif header == b'!\n': + # ar archive + return False + return False + + +def copy_arch_file(src, dst, top=None, arch_folders=()): + if os.path.isfile(src): + if top and arch_folders and is_macho_binary(src): + # Try to lipo all available archs on the first path. + src_components = src.split(os.path.sep) + top_components = top.split(os.path.sep) + if src_components[:len(top_components)] == top_components: + paths = [os.path.join(a, *(src_components[len(top_components):])) for a in arch_folders] + paths = [p for p in paths if os.path.isfile(p)] + if len(paths) > 1: + try: + run(['lipo', '-output', dst, '-create'] + paths, check=True) + except Exception: + if not is_macho_fat_binary(src): + raise + # otherwise we have two fat binaries with multiple archs + # so just copy one. + else: + return + if os.path.exists(dst): + pass # don't overwrite existing files + else: + shutil.copy2(src, dst) + + +# Modified copytree to copy new files to an existing tree. +def graft_tree(src, dst, symlinks=False, copy_function=shutil.copy2, dirs_exist_ok=False): + names = os.listdir(src) + os.makedirs(dst, exist_ok=dirs_exist_ok) + errors = [] + for name in names: + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + if os.path.exists(dstname): + continue + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + graft_tree(srcname, dstname, symlinks, copy_function, dirs_exist_ok) + else: + copy_function(srcname, dstname) + # What about devices, sockets etc.? + # catch the Error from the recursive graft_tree so that we can + # continue with other files + except shutil.Error as err: + errors.extend(err.args[0]) + except OSError as why: + errors.append((srcname, dstname, str(why))) + try: + shutil.copystat(src, dst) + except OSError as why: + # can't copy file access times on Windows + if why.winerror is None: # pylint: disable=no-member + errors.extend((src, dst, str(why))) + if errors: + raise shutil.Error(errors) + +def lipo_tree(dst_folder, arch_folders): + for folder in arch_folders: + graft_tree(folder, + dst_folder, + symlinks=True, + copy_function=lambda s, d, top=folder: copy_arch_file(s, d, + top=top, + arch_folders=arch_folders), + dirs_exist_ok=True) From f1cfab1fabfdd2c6cfcd4bf1ceae678c3d3b25ec Mon Sep 17 00:00:00 2001 From: Garrick Meeker Date: Thu, 19 Jun 2025 15:27:23 -0700 Subject: [PATCH 2/4] Avoid recursive execution of conan --- .../commands/install-universal/README.md | 10 ++- .../cmd_install_universal.py | 82 +++++++++++++------ 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/extensions/commands/install-universal/README.md b/extensions/commands/install-universal/README.md index a511d8e..a707bba 100644 --- a/extensions/commands/install-universal/README.md +++ b/extensions/commands/install-universal/README.md @@ -19,11 +19,13 @@ runs lipo when a universal package is needed. **This command is in an experimental stage, feedback is welcome.** **Parameters** -* supports all arguments used by `conan install`, see `conan install_universal --help` +* supports all arguments used by `conan install`, see `conan install-universal --help` -Note: many arguments are not correctly passed to the single architecture installs. -Currently this includes -o and -s settings. The Conan dependency graph does not support -building multiple architectures so this executes Conan again in a new process. +Note: this command builds for each architecture with the same commands and some arguments +may break something. This command currently will always load the local cache with +universal and single architecture binaries. It does not try to pull universal +binaries from a remote before handling single architectures, so this could be more +efficient in the future. **Profile** The multi-architecture must be specified in the host profile. diff --git a/extensions/commands/install-universal/cmd_install_universal.py b/extensions/commands/install-universal/cmd_install_universal.py index 949b1e7..4249cec 100644 --- a/extensions/commands/install-universal/cmd_install_universal.py +++ b/extensions/commands/install-universal/cmd_install_universal.py @@ -1,9 +1,11 @@ +import copy import json import os import shutil from subprocess import run import sys import tempfile +from contextlib import redirect_stdout from conan.api.conan_api import ConanAPI from conan.api.output import ConanOutput @@ -13,6 +15,7 @@ from conan.cli.formatters.graph import format_graph_json from conan.cli.printers import print_profiles from conan.cli.printers.graph import print_graph_packages, print_graph_basic +from conan.errors import ConanException @conan_command(group="Consumer", formatters={"json": format_graph_json}) @@ -63,6 +66,45 @@ def install_universal(conan_api: ConanAPI, parser, *args): profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) + # Build single architectures + do_universal = profile_host.settings["os"] in ["Macos", "iOS", "watchOS", "tvOS", "visionOS"] and "|" in profile_host.settings["arch"] + arch_data = {} + if do_universal: + archs = str(profile_host.settings["arch"]).split("|") + for arch in archs: + ConanOutput().title(f"Preparing {arch} binaries") + + arch_args = copy.deepcopy(args) + arch_args.settings_host = (arch_args.settings_host or []) + [f"arch={arch}"] + arch_profile_host, arch_profile_build = conan_api.profiles.get_profiles_from_args(arch_args) + print_profiles(arch_profile_host, arch_profile_build) + + # Graph computation (without installation of binaries) + gapi = conan_api.graph + if path: + deps_graph = gapi.load_graph_consumer(path, args.name, args.version, args.user, args.channel, + arch_profile_host, arch_profile_build, lockfile, remotes, + args.update, is_build_require=args.build_require) + else: + deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, arch_profile_host, + arch_profile_build, lockfile, remotes, args.update) + + print_graph_basic(deps_graph) + deps_graph.report_graph_error() + gapi.analyze_binaries(deps_graph, args.build, remotes, update=args.update, lockfile=lockfile) + print_graph_packages(deps_graph) + + # Installation of binaries and consumer generators + conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) + + with tempfile.TemporaryFile(mode="w+") as f: + with redirect_stdout(f): + format_graph_json({ + "graph": deps_graph, + "conan_api": conan_api}) + f.seek(0) + arch_data[arch] = json.load(f) + # Graph computation (without installation of binaries) gapi = conan_api.graph if path: @@ -74,10 +116,9 @@ def install_universal(conan_api: ConanAPI, parser, *args): profile_build, lockfile, remotes, args.update) # Handle universal packages - if profile_host.settings['os'] in ['Macos', 'iOS', 'watchOS', 'tvOS', 'visionOS'] and '|' in profile_host.settings['arch']: + if do_universal: for node in deps_graph.ordered_iterate(): - make_universal_conanfile(node.conanfile, args) - print(node.conanfile.name, node.conanfile.generate, node.conanfile.build) + make_universal_conanfile(node.conanfile, args, arch_data) print_graph_basic(deps_graph) deps_graph.report_graph_error() @@ -101,32 +142,23 @@ def install_universal(conan_api: ConanAPI, parser, *args): "conan_api": conan_api} -def make_universal_conanfile(conanfile, args): +def make_universal_conanfile(conanfile, args, arch_data): def _generate(conanfile): - print('generate', conanfile.name) pass def _build(conanfile): - print('build', conanfile.name) - archs = str(conanfile.settings.arch).split('|') - ref = conanfile.ref - for arch in archs: - more_args = '' - if args.profile_build: - more_args += f' -pr:b {args.profile_build[0]}' - if args.profile_host: - more_args += f' -pr:h {args.profile_host[0]}' - for arch in archs: - ConanOutput().success(f"Building universal {ref} {arch}") - arch_folder = os.path.join(conanfile.build_folder, arch) - run(f'conan install --requires={ref}{more_args} -d direct_deploy --deployer-folder "{arch_folder}" -s arch={arch} -b missing', shell=True) + pass + def _find_arch_package(conanfile, arch): + nodes = [n for n in arch_data[arch]["graph"]["nodes"].values() if n["name"] == conanfile.name and n["version"] == conanfile.version] + if len(nodes) != 1: + raise ConanException(f"Unable to find {arch} package for {conanfile.name}") + return nodes[0] def _package(conanfile): - print('package', conanfile.name) - archs = str(conanfile.settings.arch).split('|') - lipo_tree(conanfile.package_folder, [os.path.join(conanfile.build_folder, arch, 'direct_deploy', conanfile.name) for arch in archs]) - if conanfile.settings.get_safe('arch', '') and conanfile.package_type not in ('header-library', 'build-scripts', 'python-require'): - setattr(conanfile, 'generate', _generate.__get__(conanfile, type(conanfile))) - setattr(conanfile, 'build', _build.__get__(conanfile, type(conanfile))) - setattr(conanfile, 'package', _package.__get__(conanfile, type(conanfile))) + archs = str(conanfile.settings.arch).split("|") + lipo_tree(conanfile.package_folder, [_find_arch_package(conanfile, arch)["package_folder"] for arch in archs]) + if conanfile.settings.get_safe("arch", "") and conanfile.package_type not in ("header-library", "build-scripts", "python-require"): + setattr(conanfile, "generate", _generate.__get__(conanfile, type(conanfile))) + setattr(conanfile, "build", _build.__get__(conanfile, type(conanfile))) + setattr(conanfile, "package", _package.__get__(conanfile, type(conanfile))) # Lipo support From ae2763095000cf7d0477daacb1d168e2bffaa70c Mon Sep 17 00:00:00 2001 From: Garrick Meeker Date: Sat, 21 Jun 2025 10:04:52 -0700 Subject: [PATCH 3/4] Rework to use build order when building arch dependencies. --- .../commands/install-universal/README.md | 16 ++- .../cmd_install_universal.py | 105 +++++++++++------- 2 files changed, 76 insertions(+), 45 deletions(-) diff --git a/extensions/commands/install-universal/README.md b/extensions/commands/install-universal/README.md index a707bba..a471a65 100644 --- a/extensions/commands/install-universal/README.md +++ b/extensions/commands/install-universal/README.md @@ -21,11 +21,17 @@ runs lipo when a universal package is needed. **Parameters** * supports all arguments used by `conan install`, see `conan install-universal --help` -Note: this command builds for each architecture with the same commands and some arguments -may break something. This command currently will always load the local cache with -universal and single architecture binaries. It does not try to pull universal -binaries from a remote before handling single architectures, so this could be more -efficient in the future. +This is implemented with the following steps: + +1. Calculate the universal packages to build, as with `conan graph build-order` +2. Build these references for each architecture (ignoring the -b argument and building any + required dependencies). +3. Restart the install with the universal profile and -b 'never'. The recipes are replaced with + code to run lipo on the single architecture packages. + +Note: this command builds for each architecture with and some arguments +may break something. This command may load the local cache with +both universal and single architecture binaries if they need to be built. **Profile** The multi-architecture must be specified in the host profile. diff --git a/extensions/commands/install-universal/cmd_install_universal.py b/extensions/commands/install-universal/cmd_install_universal.py index 4249cec..4d086b3 100644 --- a/extensions/commands/install-universal/cmd_install_universal.py +++ b/extensions/commands/install-universal/cmd_install_universal.py @@ -66,44 +66,17 @@ def install_universal(conan_api: ConanAPI, parser, *args): profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) - # Build single architectures + args_build = args.build + + # Handle universal packages do_universal = profile_host.settings["os"] in ["Macos", "iOS", "watchOS", "tvOS", "visionOS"] and "|" in profile_host.settings["arch"] arch_data = {} if do_universal: - archs = str(profile_host.settings["arch"]).split("|") - for arch in archs: - ConanOutput().title(f"Preparing {arch} binaries") - - arch_args = copy.deepcopy(args) - arch_args.settings_host = (arch_args.settings_host or []) + [f"arch={arch}"] - arch_profile_host, arch_profile_build = conan_api.profiles.get_profiles_from_args(arch_args) - print_profiles(arch_profile_host, arch_profile_build) - - # Graph computation (without installation of binaries) - gapi = conan_api.graph - if path: - deps_graph = gapi.load_graph_consumer(path, args.name, args.version, args.user, args.channel, - arch_profile_host, arch_profile_build, lockfile, remotes, - args.update, is_build_require=args.build_require) - else: - deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, arch_profile_host, - arch_profile_build, lockfile, remotes, args.update) - - print_graph_basic(deps_graph) - deps_graph.report_graph_error() - gapi.analyze_binaries(deps_graph, args.build, remotes, update=args.update, lockfile=lockfile) - print_graph_packages(deps_graph) - - # Installation of binaries and consumer generators - conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) - - with tempfile.TemporaryFile(mode="w+") as f: - with redirect_stdout(f): - format_graph_json({ - "graph": deps_graph, - "conan_api": conan_api}) - f.seek(0) - arch_data[arch] = json.load(f) + universal = build_universal(conan_api, profile_host, profile_build, path, remotes=remotes, overrides=overrides, lockfile=lockfile, args=args) + arch_data = universal["arch_data"] + args_build = universal["refs"] + if not args_build: + args_build = ["never"] # Graph computation (without installation of binaries) gapi = conan_api.graph @@ -115,14 +88,13 @@ def install_universal(conan_api: ConanAPI, parser, *args): deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, profile_host, profile_build, lockfile, remotes, args.update) - # Handle universal packages if do_universal: for node in deps_graph.ordered_iterate(): make_universal_conanfile(node.conanfile, args, arch_data) print_graph_basic(deps_graph) deps_graph.report_graph_error() - gapi.analyze_binaries(deps_graph, args.build, remotes, update=args.update, lockfile=lockfile) + gapi.analyze_binaries(deps_graph, args_build, remotes, update=args.update, lockfile=lockfile) print_graph_packages(deps_graph) # Installation of binaries and consumer generators @@ -142,15 +114,68 @@ def install_universal(conan_api: ConanAPI, parser, *args): "conan_api": conan_api} +def build_universal(conan_api: ConanAPI, profile_host, profile_build, path, remotes, overrides, lockfile, args): + # Compute universal package build order + gapi = conan_api.graph + if path: + deps_graph = gapi.load_graph_consumer(path, args.name, args.version, args.user, args.channel, + profile_host, profile_build, lockfile, remotes, + args.update, is_build_require=args.build_require) + else: + deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, profile_host, + profile_build, lockfile, remotes, args.update) + deps_graph.report_graph_error() + gapi.analyze_binaries(deps_graph, args.build, remotes, update=args.update, lockfile=lockfile) + + install_graph = conan_api.graph.build_order(deps_graph, "recipe", True, + profile_args=args) + install_order_serialized = install_graph.install_build_order() + arch_data = {} + refs = [node["ref"] for nodes in install_order_serialized["order"] for node in nodes] + if refs: + archs = str(profile_host.settings["arch"]).split("|") + for arch in archs: + ConanOutput().title(f"Preparing {arch} binaries") + + arch_args = copy.deepcopy(args) + arch_args.settings_host = (arch_args.settings_host or []) + [f"arch={arch}"] + arch_profile_host, arch_profile_build = conan_api.profiles.get_profiles_from_args(arch_args) + print_profiles(arch_profile_host, arch_profile_build) + + # Graph computation (without installation of binaries) + arch_deps_graph = gapi.load_graph_requires(refs, [], arch_profile_host, + arch_profile_build, lockfile, remotes, args.update) + + print_graph_basic(arch_deps_graph) + arch_deps_graph.report_graph_error() + gapi.analyze_binaries(arch_deps_graph, ["missing"], remotes, update=args.update, lockfile=lockfile) + print_graph_packages(arch_deps_graph) + + # Installation of binaries and consumer generators + conan_api.install.install_binaries(deps_graph=arch_deps_graph, remotes=remotes) + + with tempfile.TemporaryFile(mode="w+") as f: + with redirect_stdout(f): + format_graph_json({ + "graph": arch_deps_graph, + "conan_api": conan_api}) + f.seek(0) + arch_data[arch] = json.load(f)["graph"]["nodes"].values() + + return { + "arch_data": arch_data, + "refs": refs} + + def make_universal_conanfile(conanfile, args, arch_data): def _generate(conanfile): pass def _build(conanfile): pass def _find_arch_package(conanfile, arch): - nodes = [n for n in arch_data[arch]["graph"]["nodes"].values() if n["name"] == conanfile.name and n["version"] == conanfile.version] - if len(nodes) != 1: - raise ConanException(f"Unable to find {arch} package for {conanfile.name}") + nodes = [n for n in arch_data[arch] if n["name"] == conanfile.name and n["version"] == conanfile.version] + if not nodes: + raise ConanException(f"Unable to find {conanfile.name} package for {arch}") return nodes[0] def _package(conanfile): archs = str(conanfile.settings.arch).split("|") From 788185c076aac6668d7bc6f8a86c137bf80ebd8c Mon Sep 17 00:00:00 2001 From: Garrick Meeker Date: Thu, 24 Jul 2025 14:39:05 -0700 Subject: [PATCH 4/4] Rework to pass options down to single architecture builds, unfortunately building more than necessary --- .../install-universal/cmd_install_universal.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extensions/commands/install-universal/cmd_install_universal.py b/extensions/commands/install-universal/cmd_install_universal.py index 4d086b3..e96b3be 100644 --- a/extensions/commands/install-universal/cmd_install_universal.py +++ b/extensions/commands/install-universal/cmd_install_universal.py @@ -143,8 +143,14 @@ def build_universal(conan_api: ConanAPI, profile_host, profile_build, path, remo print_profiles(arch_profile_host, arch_profile_build) # Graph computation (without installation of binaries) - arch_deps_graph = gapi.load_graph_requires(refs, [], arch_profile_host, - arch_profile_build, lockfile, remotes, args.update) + if path: + # TODO: limit builds that are not necessary for universal + arch_deps_graph = gapi.load_graph_consumer(path, args.name, args.version, args.user, args.channel, + arch_profile_host, arch_profile_build, lockfile, remotes, + False, is_build_require=args.build_require) + else: + arch_deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, arch_profile_host, + arch_profile_build, lockfile, remotes, False) print_graph_basic(arch_deps_graph) arch_deps_graph.report_graph_error()