Add a repository rule for efficient sysroots

Author: dzbarsky

MODULE.bazel

@@ -21,9 +21,18 @@ module(
 
 bazel_dep(name = "bazel_features", version = "1.36.0")
 bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "aspect_bazel_lib", version = "2.0.0")
 bazel_dep(name = "rules_cc", version = "0.2.2")
 bazel_dep(name = "platforms", version = "0.0.8")
 bazel_dep(name = "helly25_bzl", version = "0.1.2")
 
+bazel_dep(name = "tar.bzl", version = "0.6.0")
+tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains")
+use_repo(tar_toolchains,
+    "bsd_tar_toolchains_darwin_arm64",
+    "bsd_tar_toolchains_linux_arm64",
+    "bsd_tar_toolchains_linux_amd64",
+)
+
 # TODO: Remove when protobuf is released with a version of rules_python that supports 8.x
 bazel_dep(name = "rules_python", version = "1.0.0", dev_dependency = True)

tests/MODULE.bazel

@@ -171,7 +171,7 @@ llvm.toolchain_root(
 )
 llvm.sysroot(
     name = "llvm_toolchain_with_sysroot",
-    label = "@org_chromium_sysroot_linux_x64//:sysroot",
+    label = "@org_chromium_sysroot_linux_x64//sysroot",
     targets = ["linux-x86_64"],
 )
 use_repo(llvm, "llvm_toolchain_with_sysroot")
@@ -253,18 +253,11 @@ libclang_rt_wasm32 = use_repo_rule("//wasm:wasi_sdk.bzl", "libclang_rt_wasm32")
 
 libclang_rt_wasm32(name = "libclang_rt_wasm32")
 
-http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+sysroot = use_repo_rule("@toolchains_llvm//toolchain:sysroot.bzl", "sysroot")
 
 # This sysroot is used by github.com/vsco/bazel-toolchains.
-http_archive(
+sysroot(
     name = "org_chromium_sysroot_linux_x64",
-    build_file_content = """
-filegroup(
-  name = "sysroot",
-  srcs = glob(["*/**"]),
-  visibility = ["//visibility:public"],
-)
-""",
     sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
-    urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
+    url = "https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz",
 )

tests/WORKSPACE

@@ -114,24 +114,19 @@ llvm_toolchain(
 ## Toolchain example with a sysroot.
 
 # This sysroot is used by github.com/vsco/bazel-toolchains.
-http_archive(
+load("@toolchains_llvm//toolchain:sysroot.bzl", "sysroot")
+
+sysroot(
     name = "org_chromium_sysroot_linux_x64",
-    build_file_content = """
-filegroup(
-  name = "sysroot",
-  srcs = glob(["*/**"]),
-  visibility = ["//visibility:public"],
-)
-""",
     sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
-    urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
+    url = "https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz",
 )
 
 llvm_toolchain(
     name = "llvm_toolchain_with_sysroot",
     llvm_versions = LLVM_VERSIONS,
     sysroot = {
-        "linux-x86_64": "@org_chromium_sysroot_linux_x64//:sysroot",
+        "linux-x86_64": "@org_chromium_sysroot_linux_x64//sysroot",
     },
     # We can share the downloaded LLVM distribution with the first configuration.
     toolchain_roots = {

toolchain/sysroot.bzl

@@ -0,0 +1,50 @@
+load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils")
+
+def _sysroot_impl(rctx):
+    url = rctx.attr.url
+    _, _, archive = url.rpartition("/")
+
+    rctx.download(url, archive, sha256 = rctx.attr.sha256)
+
+    # Declare the sysroot files as a source directory so they can be
+    # optimized in the Merkle tree cache more effectively.
+    rctx.file("sysroot/BUILD.bazel", \
+"""filegroup(
+    name = "sysroot",
+    srcs = ["."],
+    visibility = ["//visibility:public"],
+)""")
+
+    host_bsdtar = Label("@bsd_tar_toolchains_%s//:tar" % repo_utils.platform(rctx))
+
+    cmd = [
+        str(rctx.path(host_bsdtar)),
+        "--extract",
+        "--no-same-owner",
+        "--no-same-permissions",
+        "--file", archive,
+        "--directory", "sysroot",
+    ]
+
+    for include in rctx.attr.include_patterns:
+        cmd.extend(["--include", include])
+
+    for exclude in rctx.attr.exclude_patterns:
+        cmd.extend(["--exclude", exclude])
+
+    result = rctx.execute(cmd)
+    if result.return_code != 0:
+        fail(result.stdout + result.stderr)
+
+    rctx.delete(archive)
+
+sysroot = repository_rule(
+    implementation = _sysroot_impl,
+    attrs = {
+        "url": attr.string(),
+        "sha256": attr.string(),
+        "include_patterns": attr.string_list(),
+        "exclude_patterns": attr.string_list(),
+    },
+)
+