Command for support of macOS/iOS universal binaries

extensions/commands/install-universal/README.md

@@ -0,0 +1,59 @@
+## [Install macOS/iOS universal binaries](cmd_install_universal.py)
+
+Install dependent packages as universal binaries packages.
+Conan v2 introduces support for multiple architectures, e.g. -s arch=armv8|x86_64
+however this is largely limited to CMake.
+
+For CMake-based project that only depends on other CMake-based recipes, it's now
+possible to run:
+
+```
+conan install . -pr:h universal -pr:b default -b missing
+conan build . -pr:h universal -pr:b default
+```
+
+However, many other recipes use autotools or other build systems that don't support
+universal binaries. This command skips the usual build() / package() steps and
+runs lipo when a universal package is needed.
+
+**This command is in an experimental stage, feedback is welcome.**
+
+**Parameters**
+* supports all arguments used by `conan install`, see `conan install_universal --help`
+
+Note: many arguments are not correctly passed to the single architecture installs.
+Currently this includes -o and -s settings. The Conan dependency graph does not support
+building multiple architectures so this executes Conan again in a new process.
+
+**Profile**
+The multi-architecture must be specified in the host profile.
+The build profile will likely be a single architecture, although the default
+binary compatibility plugin does not know that universal binaries can be used
+for single architecture profiles.
+
+```
+[settings]
+arch=armv8|x86_64
+build_type=Release
+compiler=apple-clang
+compiler.cppstd=17
+compiler.libcxx=libc++
+compiler.version=15
+os=Macos
+os.version=11.0
+```
+
+Also make sure to use += not = if you define CXXFLAGS or you will override the -arch flag
+to autotools.
+
+```
+[buildenv]
+CFLAGS+=-fvisibility=hidden
+CXXFLAGS+=-fvisibility=hidden -fvisibility-inlines-hidden
+```
+
+Usage:
+```
+conan install-universal . -pr:h universal -pr:b default -b missing
+conan build . -pr:h universal -pr:b default
+```

extensions/commands/install-universal/cmd_install_universal.py

@@ -0,0 +1,248 @@
+import json
+import os
+import shutil
+from subprocess import run
+import sys
+import tempfile
+
+from conan.api.conan_api import ConanAPI
+from conan.api.output import ConanOutput
+from conan.cli import make_abs_path
+from conan.cli.args import common_graph_args, validate_common_graph_args
+from conan.cli.command import conan_command
+from conan.cli.formatters.graph import format_graph_json
+from conan.cli.printers import print_profiles
+from conan.cli.printers.graph import print_graph_packages, print_graph_basic
+
+
+@conan_command(group="Consumer", formatters={"json": format_graph_json})
+def install_universal(conan_api: ConanAPI, parser, *args):
+    """
+    Install universal packages from the requirements specified in a recipe (conanfile.py or conanfile.txt).
+
+    It can also be used to install packages without a conanfile, using the
+    --requires and --tool-requires arguments.
+
+    If any requirement is not found in the local cache, it will iterate the remotes
+    looking for it. When the full dependency graph is computed, and all dependencies
+    recipes have been found, it will look for binary packages matching the current settings.
+    If no binary package is found for some or several dependencies, it will error,
+    unless the '--build' argument is used to build it from source.
+
+    After installation of packages, the generators and deployers will be called.
+    """
+    common_graph_args(parser)
+    parser.add_argument("-g", "--generator", action="append", help='Generators to use')
+    parser.add_argument("-of", "--output-folder",
+                        help='The root output folder for generated and build files')
+    parser.add_argument("-d", "--deployer", action="append",
+                        help="Deploy using the provided deployer to the output folder. "
+                             "Built-in deployers: 'full_deploy', 'direct_deploy', 'runtime_deploy'")
+    parser.add_argument("--deployer-folder",
+                        help="Deployer output folder, base build folder by default if not set")
+    parser.add_argument("--deployer-package", action="append",
+                        help="Execute the deploy() method of the packages matching "
+                             "the provided patterns")
+    parser.add_argument("--build-require", action='store_true', default=False,
+                        help='Whether the provided path is a build-require')
+    parser.add_argument("--envs-generation", default=None, choices=["false"],
+                        help="Generation strategy for virtual environment files for the root")
+    args = parser.parse_args(*args)
+    validate_common_graph_args(args)
+    # basic paths
+    cwd = os.getcwd()
+    path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None
+    source_folder = os.path.dirname(path) if args.path else cwd
+    output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None
+
+    # Basic collaborators: remotes, lockfile, profiles
+    remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
+    lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd,
+                                               partial=args.lockfile_partial, overrides=overrides)
+    profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
+    print_profiles(profile_host, profile_build)
+
+    # Graph computation (without installation of binaries)
+    gapi = conan_api.graph
+    if path:
+        deps_graph = gapi.load_graph_consumer(path, args.name, args.version, args.user, args.channel,
+                                              profile_host, profile_build, lockfile, remotes,
+                                              args.update, is_build_require=args.build_require)
+    else:
+        deps_graph = gapi.load_graph_requires(args.requires, args.tool_requires, profile_host,
+                                              profile_build, lockfile, remotes, args.update)
+
+    # Handle universal packages
+    if profile_host.settings['os'] in ['Macos', 'iOS', 'watchOS', 'tvOS', 'visionOS'] and '|' in profile_host.settings['arch']:
+        for node in deps_graph.ordered_iterate():
+            make_universal_conanfile(node.conanfile, args)
+            print(node.conanfile.name, node.conanfile.generate, node.conanfile.build)
+
+    print_graph_basic(deps_graph)
+    deps_graph.report_graph_error()
+    gapi.analyze_binaries(deps_graph, args.build, remotes, update=args.update, lockfile=lockfile)
+    print_graph_packages(deps_graph)
+
+    # Installation of binaries and consumer generators
+    conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes)
+    ConanOutput().title("Finalizing install (deploy, generators)")
+    conan_api.install.install_consumer(deps_graph, args.generator, source_folder, output_folder,
+                                       deploy=args.deployer, deploy_package=args.deployer_package,
+                                       deploy_folder=args.deployer_folder,
+                                       envs_generation=args.envs_generation)
+    ConanOutput().success("Install finished successfully")
+
+    # Update lockfile if necessary
+    lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages,
+                                                  clean=args.lockfile_clean)
+    conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd)
+    return {"graph": deps_graph,
+            "conan_api": conan_api}
+
+
+def make_universal_conanfile(conanfile, args):
+    def _generate(conanfile):
+        print('generate', conanfile.name)
+        pass
+    def _build(conanfile):
+        print('build', conanfile.name)
+        archs = str(conanfile.settings.arch).split('|')
+        ref = conanfile.ref
+        for arch in archs:
+            more_args = ''
+            if args.profile_build:
+                more_args += f' -pr:b {args.profile_build[0]}'
+            if args.profile_host:
+                more_args += f' -pr:h {args.profile_host[0]}'
+            for arch in archs:
+                ConanOutput().success(f"Building universal {ref} {arch}")
+                arch_folder = os.path.join(conanfile.build_folder, arch)
+                run(f'conan install --requires={ref}{more_args} -d direct_deploy --deployer-folder "{arch_folder}" -s arch={arch} -b missing', shell=True)
+    def _package(conanfile):
+        print('package', conanfile.name)
+        archs = str(conanfile.settings.arch).split('|')
+        lipo_tree(conanfile.package_folder, [os.path.join(conanfile.build_folder, arch, 'direct_deploy', conanfile.name) for arch in archs])
+    if conanfile.settings.get_safe('arch', '') and conanfile.package_type not in ('header-library', 'build-scripts', 'python-require'):
+        setattr(conanfile, 'generate', _generate.__get__(conanfile, type(conanfile)))
+        setattr(conanfile, 'build', _build.__get__(conanfile, type(conanfile)))
+        setattr(conanfile, 'package', _package.__get__(conanfile, type(conanfile)))
+
+
+# Lipo support
+
+# These are for optimization only, to avoid unnecessarily reading files.
+_binary_exts = ['.a', '.dylib']
+_regular_exts = [
+    '.h', '.hpp', '.hxx', '.c', '.cc', '.cxx', '.cpp', '.m', '.mm', '.txt', '.md', '.html', '.jpg', '.png', '.class'
+]
+
+
+def is_macho_binary(filename):
+    ext = os.path.splitext(filename)[1]
+    if ext in _binary_exts:
+        return True
+    if ext in _regular_exts:
+        return False
+    with open(filename, "rb") as f:
+        header = f.read(4)
+        if header == b'\xcf\xfa\xed\xfe':
+            # cffaedfe is Mach-O binary
+            return True
+        elif header == b'\xca\xfe\xba\xbe':
+            # cafebabe is Mach-O fat binary
+            return True
+        elif header == b'!<arch>\n':
+            # ar archive
+            return True
+    return False
+
+
+def is_macho_fat_binary(filename):
+    ext = os.path.splitext(filename)[1]
+    if ext in _binary_exts:
+        return True
+    if ext in _regular_exts:
+        return False
+    with open(filename, "rb") as f:
+        header = f.read(4)
+        if header == b'\xcf\xfa\xed\xfe':
+            # cffaedfe is Mach-O binary
+            return False
+        elif header == b'\xca\xfe\xba\xbe':
+            # cafebabe is Mach-O fat binary
+            return True
+        elif header == b'!<arch>\n':
+            # ar archive
+            return False
+    return False
+
+
+def copy_arch_file(src, dst, top=None, arch_folders=()):
+    if os.path.isfile(src):
+        if top and arch_folders and is_macho_binary(src):
+            # Try to lipo all available archs on the first path.
+            src_components = src.split(os.path.sep)
+            top_components = top.split(os.path.sep)
+            if src_components[:len(top_components)] == top_components:
+                paths = [os.path.join(a, *(src_components[len(top_components):])) for a in arch_folders]
+                paths = [p for p in paths if os.path.isfile(p)]
+                if len(paths) > 1:
+                    try:
+                        run(['lipo', '-output', dst, '-create'] + paths, check=True)
+                    except Exception:
+                        if not is_macho_fat_binary(src):
+                            raise
+                        # otherwise we have two fat binaries with multiple archs
+                        # so just copy one.
+                    else:
+                        return
+        if os.path.exists(dst):
+            pass # don't overwrite existing files
+        else:
+            shutil.copy2(src, dst)
+
+
+# Modified copytree to copy new files to an existing tree.
+def graft_tree(src, dst, symlinks=False, copy_function=shutil.copy2, dirs_exist_ok=False):
+    names = os.listdir(src)
+    os.makedirs(dst, exist_ok=dirs_exist_ok)
+    errors = []
+    for name in names:
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if symlinks and os.path.islink(srcname):
+                if os.path.exists(dstname):
+                    continue
+                linkto = os.readlink(srcname)
+                os.symlink(linkto, dstname)
+            elif os.path.isdir(srcname):
+                graft_tree(srcname, dstname, symlinks, copy_function, dirs_exist_ok)
+            else:
+                copy_function(srcname, dstname)
+            # What about devices, sockets etc.?
+        # catch the Error from the recursive graft_tree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        shutil.copystat(src, dst)
+    except OSError as why:
+        # can't copy file access times on Windows
+        if why.winerror is None: # pylint: disable=no-member
+            errors.extend((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+
+def lipo_tree(dst_folder, arch_folders):
+    for folder in arch_folders:
+        graft_tree(folder,
+                   dst_folder,
+                   symlinks=True,
+                   copy_function=lambda s, d, top=folder: copy_arch_file(s, d,
+                                                                         top=top,
+                                                                         arch_folders=arch_folders),
+                   dirs_exist_ok=True)