From b23774b3d3749c73d58ff0f8a775fefefaf65fb9 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 26 Apr 2022 18:53:24 +0200 Subject: [PATCH 01/14] raw-efi: add /boot partition --- formats/raw-efi.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/formats/raw-efi.nix b/formats/raw-efi.nix index f185a913..4146903f 100644 --- a/formats/raw-efi.nix +++ b/formats/raw-efi.nix @@ -8,6 +8,11 @@ efiInstallAsRemovable = true; }; + fileSystems."/boot" = { + device = "/dev/vda1"; + fsType = "vfat"; + }; + system.build.raw = lib.mkOverride 999 (import "${toString modulesPath}/../lib/make-disk-image.nix" { inherit lib config pkgs; partitionTableType = "efi"; From b2296439aed6ae5ae8a0d1bb7170e82e671df047 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 26 Apr 2022 19:09:36 +0200 Subject: [PATCH 02/14] raw-efi: override build.raw harder --- formats/raw-efi.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/raw-efi.nix b/formats/raw-efi.nix index 4146903f..e566e0af 100644 --- a/formats/raw-efi.nix +++ b/formats/raw-efi.nix @@ -13,7 +13,7 @@ fsType = "vfat"; }; - system.build.raw = lib.mkOverride 999 (import "${toString modulesPath}/../lib/make-disk-image.nix" { + system.build.raw = lib.mkOverride 99 (import "${toString modulesPath}/../lib/make-disk-image.nix" { inherit lib config pkgs; partitionTableType = "efi"; diskSize = "auto"; From 7c4d5b2048c346bef056588f2a0acb37c4c1f827 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 26 Apr 2022 19:09:49 +0200 Subject: [PATCH 03/14] vm-nogui: fix syntax error --- formats/vm-nogui.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/vm-nogui.nix b/formats/vm-nogui.nix index b5b627be..449baf91 100644 --- a/formats/vm-nogui.nix +++ b/formats/vm-nogui.nix @@ -8,7 +8,7 @@ let IFS=';t' read -r _ rows cols _ < /dev/tty stty "$old" stty cols "$cols" rows "$rows" - ] + fi ''; # https://unix.stackexchange.com/questions/16578/resizable-serial-console-window in { imports = [ From 4483147d467f29391ab37526810b25a5bf5208bf Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Fri, 13 May 2022 11:44:53 -0400 Subject: [PATCH 04/14] Resolve links when finding output filename by s/-type/-xtype/ in the "find" invocation. Now nixos-generate prints both regular files and symlinks to regular files (like "run-nixos-vm" for the "vm" and "vm-nogui" formats). --- nixos-generate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-generate b/nixos-generate index 40a0151c..04dc910e 100755 --- a/nixos-generate +++ b/nixos-generate @@ -169,7 +169,7 @@ out=$(nix-build "${nix_args[@]}" "${nix_build_args[@]}" -A "config.system.build. if [[ -z $run ]]; then # show the first file, ignoring nix-support - find "$out" -wholename "$filename" -type f -print -quit + find "$out" -wholename "$filename" -xtype f -print -quit else runner=$(find "$out"/bin -type l -print -quit) exec "$runner" From b0ddf4ec84c555fe140899aadeb8c51b99876aac Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Fri, 13 May 2022 11:45:22 -0400 Subject: [PATCH 05/14] CI: add nixos-generate logging output assertions Namely, assert that the last line of output refers to a store path that is a file or symlink to a file. --- .included.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.included.yml b/.included.yml index 46055608..e7926c55 100644 --- a/.included.yml +++ b/.included.yml @@ -22,6 +22,9 @@ stages: - mkdir $uploadPath script: - nix-shell --command './nixos-generate -f $formats -I nixpkgs=channel:nixos-$nixpkgs_ver' |& tee $uploadPath/build.log + # Last line of output should be the path of a regular file (or symlink + # to a regular file) in the Nix store. + - storePath=$(tail -n 1 $uploadPath/build.log) && test -f "$storePath" && test "/nix/store/${storePath#/nix/store/}" = "$storePath" - mv $(realpath result) $uploadPath artifacts: name: "$uploadPath" From 788cdc592081c2ab3f6a40cf8cff79b3b16542ad Mon Sep 17 00:00:00 2001 From: Simon Elsbrock Date: Tue, 17 May 2022 22:47:23 +0200 Subject: [PATCH 06/14] docs: fix wrong sentence for `-o` --- nixos-generate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-generate b/nixos-generate index 04dc910e..d3adbb24 100755 --- a/nixos-generate +++ b/nixos-generate @@ -41,7 +41,7 @@ Options: * --system: specify the target system (eg: x86_64-linux) * -o, --out-link: specify the outlink location for nix-build * --cores : to control the maximum amount of parallelism. (see nix-build documentation) -* --option : Passed to to nix-build (see nix-build documentation). +* --option : Passed to nix-build (see nix-build documentation). * -I KEY=VALUE: Add a key to the Nix expression search path. USAGE } From 24b95e336e0593e551488c1c2a7ffe82ab898ee7 Mon Sep 17 00:00:00 2001 From: Samuel Tam Date: Tue, 24 May 2022 00:44:07 +0800 Subject: [PATCH 07/14] README.md: update cross compiling section --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 018af195..0178c326 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,52 @@ NIX_PATH=nixpkgs=../nixpkgs nixos-generate -f do ## Cross Compiling -To cross compile nixos images for other system you have -to configure `boot.binfmtMiscRegistrations` on your host system. +To cross compile nixos images for other architectures you have to configure +`boot.binfmt.emulatedSystems` or `boot.binfmt.registrations` on your host system. -For more details about this have a look at : +In your system `configuration.nix`: +```nix +{ + # Enable binfmt emulation of aarch64-linux. + boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; +} +``` + +Alternatively, if you want to target other architectures: +```nix +# Define qemu-arm-static source. +let qemu-arm-static = pkgs.stdenv.mkDerivation { + name = "qemu-arm-static"; + src = builtins.fetchurl { + url = "https://github.com/multiarch/qemu-user-static/releases/download/v6.1.0-8/qemu-arm-static"; + sha256 = "06344d77d4f08b3e1b26ff440cb115179c63ca8047afb978602d7922a51231e3"; + }; + dontUnpack = true; + installPhase = "install -D -m 0755 $src $out/bin/qemu-arm-static"; +}; +in { + # Enable binfmt emulation of extra binary formats (armv7l-linux, for exmaple). + boot.binfmt.registrations.arm = { + interpreter = "${qemu-arm-static}/bin/qemu-arm-static"; + magicOrExtension = ''\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00''; + mask = ''\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\x00\xff\xfe\xff\xff\xff''; + }; + + # Define additional settings for nix. + nix.extraOptions = '' + extra-platforms = armv7l-linux + ''; + nix.sandboxPaths = [ "/run/binfmt/arm=${qemu-arm-static}/bin/qemu-arm-static" ]; +} +``` + +For more details on configuring `binfmt`, have a look at: +[binfmt options](https://search.nixos.org/options?channel=unstable&query=boot.binfmt), +[binfmt.nix](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/system/boot/binfmt.nix), +[this comment](https://github.com/NixOS/nixpkgs/issues/109661#issuecomment-762629438) and [clevers qemu-user](https://github.com/cleverca22/nixos-configs/blob/master/qemu.nix). -Once you've run `nixos-rebuild` with theses options, +Once you've run `nixos-rebuild` with these options, you can use the `--system` option to create images for other architectures. ## Using in a Flake From 494ceb7a63f7548e18135eb57ae1a1aab632f9e3 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Mon, 16 May 2022 20:38:22 -0400 Subject: [PATCH 08/14] Nix >= 2.7 app..default compat --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 6fb163d6..136e8d3e 100644 --- a/flake.nix +++ b/flake.nix @@ -74,12 +74,19 @@ buildInputs = with pkgs; [ jq coreutils findutils ]; }); - # Make it runnable with `nix app` - apps = forAllSystems (system: { + # Make it runnable with `nix run` + apps = forAllSystems (system: let nixos-generate = { type = "app"; program = "${self.packages."${system}".nixos-generators}/bin/nixos-generate"; }; + in { + inherit nixos-generate; + + # Nix >= 2.7 flake output schema uses `apps..default` instead + # of `defaultApp.` to signify the default app (the thing that + # gets run with `nix run . -- `) + default = nixos-generate; }); defaultApp = forAllSystems (system: self.apps."${system}".nixos-generate); From 86bf2a608b3149492b60b9b15c4e864149e15a62 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Tue, 17 May 2022 14:36:40 -0400 Subject: [PATCH 09/14] kexec-bundle: define filename pattern --- formats/kexec-bundle.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/formats/kexec-bundle.nix b/formats/kexec-bundle.nix index e18bce0f..e2878d76 100644 --- a/formats/kexec-bundle.nix +++ b/formats/kexec-bundle.nix @@ -3,4 +3,5 @@ imports = [ ./kexec.nix ]; formatAttr = lib.mkForce "kexec_bundle"; + filename = lib.mkForce "*-kexec_bundle"; } From f8773d5e9c663da94d6fa0eac3cc7409199ab7d7 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Tue, 17 May 2022 12:15:45 -0400 Subject: [PATCH 10/14] No lib.mkForce in system.build.kexec_tarball def'n on nixpkgs versions prior to the commit that changed config.system.build's type from a lazy attribute set to a submodule. Prior to this type change, there is no system.build.kexec_tarball option declared, so the NixOS module system does not resolve priorities/overrides in the config.build.kexec_tarball definition. That is, with lib.mkForce, the config.build.kexec_tarball definition ends up being something like: { _type = "override"; content = <...>; priority = 50; } Removing lib.mkForce allows us to successfully and sensibly interpolate the value (== outPath) of system.build.kexec_tarball in system.build.kexec_bundle's builder script. Likewise, no lib.mkOverride for system.build.raw. --- Makefile | 2 +- formats/kexec.nix | 6 ++++-- formats/raw-efi.nix | 9 ++++++--- lib.nix | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 lib.nix diff --git a/Makefile b/Makefile index 439bfe04..50acd4e2 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SHARE ?= $(PREFIX)/share/nixos-generator all: -SOURCES = formats format-module.nix configuration.nix nixos-generate.nix +SOURCES = formats format-module.nix configuration.nix lib.nix nixos-generate.nix install: mkdir -p $(PREFIX)/bin $(SHARE) diff --git a/formats/kexec.nix b/formats/kexec.nix index a124454d..47878fbb 100644 --- a/formats/kexec.nix +++ b/formats/kexec.nix @@ -1,9 +1,11 @@ -{ config, pkgs, lib, modulesPath, ... }: let +{ config, pkgs, lib, modulesPath, options, ... }: let clever-tests = builtins.fetchGit { url = "https://github.com/cleverca22/nix-tests"; rev = "a9a316ad89bfd791df4953c1a8b4e8ed77995a18"; # master on 2021-06-13 }; + + inherit (import ../lib.nix { inherit lib options; }) maybe; in { imports = [ "${toString modulesPath}/installer/netboot/netboot-minimal.nix" @@ -13,7 +15,7 @@ in { ]; system.build = rec { - kexec_tarball = lib.mkForce (pkgs.callPackage "${toString modulesPath}/../lib/make-system-tarball.nix" { + kexec_tarball = maybe.mkForce (pkgs.callPackage "${toString modulesPath}/../lib/make-system-tarball.nix" { storeContents = [ { object = config.system.build.kexec_script; symlink = "/kexec_nixos"; } ]; diff --git a/formats/raw-efi.nix b/formats/raw-efi.nix index e566e0af..6ebabad7 100644 --- a/formats/raw-efi.nix +++ b/formats/raw-efi.nix @@ -1,5 +1,8 @@ -{ config, lib, pkgs, modulesPath, ... }: -{ +{ config, lib, options, pkgs, modulesPath, ... }: + +let + inherit (import ../lib.nix { inherit lib options; }) maybe; +in { imports = [ ./raw.nix ]; boot.loader.grub = { @@ -13,7 +16,7 @@ fsType = "vfat"; }; - system.build.raw = lib.mkOverride 99 (import "${toString modulesPath}/../lib/make-disk-image.nix" { + system.build.raw = maybe.mkOverride 99 (import "${toString modulesPath}/../lib/make-disk-image.nix" { inherit lib config pkgs; partitionTableType = "efi"; diskSize = "auto"; diff --git a/lib.nix b/lib.nix new file mode 100644 index 00000000..83e365e0 --- /dev/null +++ b/lib.nix @@ -0,0 +1,32 @@ +{ + lib, + options, +}: let + # See https://github.com/NixOS/nixpkgs/commit/ccb85a53b6a496984073227fd8c4d4c58889f421 + # This commit changed the type of `system.build` from a lazy attribute set to + # a submodule. Prior to this commit, it doesn't make sense to qualify, e.g. + # the `system.build.kexec_tarball` definition with `lib.mkForce`, as this + # would result in setting the (final/resolved) value of + # `system.build.kexec_tarball` to something like: + # { + # _type = "override"; + # content = <...>; + # priority = 50; + # } + # However, since this commit, `system.build.kexec_tarball` *must* be defined + # using `lib.mkForce`; otherwise, Nix bails out with a complaint about + # `system.build.kexec_tarball` being defined in multiple locations. + systemBuildIsSubmodule = options.system.build.type.name == "submodule"; + + optionsLookSane = lib.hasAttrByPath ["system" "build" "type" "name"] options; +in + assert (lib.assertMsg optionsLookSane "`options' must be the NixOS module `options' argument"); { + maybe = + { + mkForce = lib.id; + mkOverride = _: lib.id; + } + // (lib.optionalAttrs systemBuildIsSubmodule { + inherit (lib) mkForce mkOverride; + }); + } From 80365c58679b3bdb6a22b3be9c819021382aed25 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Wed, 18 May 2022 09:22:57 -0400 Subject: [PATCH 11/14] flake.lock: update nixpkgs input to gain access to proxmox-lxc and kubevirt profiles --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 8cdc03b5..aedd00b7 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1637186689, - "narHash": "sha256-NU7BhgnwA/3ibmCeSzFK6xGi+Bari9mPfn+4cBmyEjw=", + "lastModified": 1652739558, + "narHash": "sha256-znGkjGugajqF/sFS+H4+ENmGTaVPFE0uu1JjQZJLEaQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7fad01d9d5a3f82081c00fb57918d64145dc904c", + "rev": "ff691ed9ba21528c1b4e034f36a04027e4522c58", "type": "github" }, "original": { From 5f1a99a2dace7ff140da4b7e4a3b38e10a1d972e Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Fri, 20 May 2022 13:01:40 -0400 Subject: [PATCH 12/14] Add flake checks that build various images. Limitation: checks only exist for x86_64-linux and aarch64-linux. Incidental changes: 1. Add nixos-21.11 release as a flake input. 2. Add the function `nixosGenerate'`, which is a generalized version of `nixosGenerate` that takes the additional mandatory argument `system` and the additional optional argument `nixpkgs` (the latter of which should be a nixpkgs *flake*). This function is used for constructing the flake checks; `nixpkgs` needs to be parameterized in order to use both nixpkgs-unstable and nixos-21.11. --- flake.lock | 17 +++++++ flake.nix | 133 +++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 13 deletions(-) diff --git a/flake.lock b/flake.lock index aedd00b7..1aef49a2 100644 --- a/flake.lock +++ b/flake.lock @@ -15,6 +15,22 @@ "type": "github" } }, + "nixos": { + "locked": { + "lastModified": 1652881001, + "narHash": "sha256-k9JmPCojaJnqGz4aRXXT1HZqJKHCXijoMfBAb24abXk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2d474d6a4a43a0348b78db68dc00c491032cf5cf", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-21.11", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1652739558, @@ -34,6 +50,7 @@ "root": { "inputs": { "nixlib": "nixlib", + "nixos": "nixos", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index 136e8d3e..9cac5314 100644 --- a/flake.nix +++ b/flake.nix @@ -6,8 +6,9 @@ # Bin dependency inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + inputs.nixos.url = "github:NixOS/nixpkgs/nixos-21.11"; - outputs = { self, nixpkgs, nixlib }: + outputs = { self, nixpkgs, nixos, nixlib }@inputs: # Library modules (depend on nixlib) rec { @@ -18,16 +19,15 @@ value.imports = [ (./formats + "/${file}") ./format-module.nix ]; }) (builtins.readDir ./formats); - # example usage in flakes: - # outputs = { self, nixpkgs, nixos-generators, ...}: { - # vmware = nixos-generators.nixosGenerate { - # pkgs = nixpkgs.legacyPackages.x86_64-linux; - # modules = [./configuration.nix]; - # format = "vmware"; - # }; - # } - nixosGenerate = { pkgs, format, specialArgs ? { }, modules ? [ ] }: - let + nixosGenerate' = { + format + , system + , nixpkgs ? inputs.nixpkgs + , pkgs ? nixpkgs.legacyPackages.${system} + , specialArgs ? { } + , modules ? [ ] + }: + let formatModule = builtins.getAttr format nixosModules; image = nixpkgs.lib.nixosSystem { inherit pkgs specialArgs; @@ -36,14 +36,26 @@ formatModule ] ++ modules; }; - in + in assert system == pkgs.system; image.config.system.build.${image.config.formatAttr}; + # example usage in flakes: + # outputs = { self, nixpkgs, nixos-generators, ...}: { + # vmware = nixos-generators.nixosGenerate { + # pkgs = nixpkgs.legacyPackages.x86_64-linux; + # modules = [./configuration.nix]; + # format = "vmware"; + # }; + # } + nixosGenerate = { pkgs, format, specialArgs ? { }, modules ? [ ] }: + nixosGenerate' { + system = pkgs.system; + inherit pkgs format specialArgs modules; + }; } // - # Binary and Devshell outputs (depend on nixpkgs) ( let @@ -90,6 +102,101 @@ }); defaultApp = forAllSystems (system: self.apps."${system}".nixos-generate); + + checks = let + # No way to limit `nix flake check` to a subset of supported systems; + # see https://github.com/NixOS/nix/issues/6398. + forCheckSystems = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ]; + + checksForNixpkgs = + system: + { + input, + id, + modules ? (fetchModules system id) + }: + let + pkgs = input.legacyPackages.${system}; + generateFormat = format: self.nixosGenerate' { + inherit format system pkgs; + nixpkgs = input; + }; + #formatModules = builtins.removeAttrs self.nixosModules exclude; + in nixlib.lib.mapAttrs' (format: _: let + name = "${format}-${id}"; + diag = ''evaluating format "${format}" using nixpkgs input "${id}" on system "${system}"''; + value = nixlib.lib.trace diag (generateFormat format); + in { + inherit name value; + }) modules; + + fetchModules = let + excludeCommon = [ + # error: + # Failed assertions: + # - Mountpoint '/': 'autoResize = true' is not supported for 'fsType = "auto"': fsType has to be explicitly set and only the ext filesystems and f2fs support it. + "cloudstack" + + # error (ignored): error: cannot look up '' in pure evaluation mode (use '--impure' to override) + # + # at /nix/store/0b099b46lb9dmhwzyc2zgjk8lp8d9rfq-source/kexec/kexec.nix:42:49: + # + # 41| ''; + # 42| system.build.kexec_tarball = pkgs.callPackage { + # | ^ + # 43| storeContents = [ + "kexec" + "kexec-bundle" + ]; + + excludeNixpkgs = [ + # error: path '/nix/store/0bsydzh62cn1by07j5cjy28crbnbc5wz-google-guest-configs-20211116.00.drv' is not valid) + "gce" + + # Compilation error in some prerequisite or other. + "proxmox" + ]; + + excludeNixos = [ + # error: getting status of '/nix/store/<...>/nixos/modules/virtualisation/kubevirt.nix': No such file or directory + "kubevirt" + + # error: getting status of '/nix/store/<...>/nixos/modules/virtualisation/proxmox-lxc.nix': No such file or directory + "proxmox-lxc" + ]; + + aarch64Only = [ "sd-aarch64" "sd-aarch64-installer" ]; + + baseModules = builtins.removeAttrs self.nixosModules excludeCommon; + nixpkgsModules = builtins.removeAttrs baseModules excludeNixpkgs; + nixosModules = builtins.removeAttrs baseModules excludeNixos; + + matrix = { + "x86_64-linux" = { + "nixpkgs" = builtins.removeAttrs nixpkgsModules aarch64Only; + "nixos" = builtins.removeAttrs nixosModules aarch64Only; + }; + + "aarch64-linux" = { + "nixpkgs" = nixlib.lib.getAttrs aarch64Only nixpkgsModules; + "nixos" = nixlib.lib.getAttrs aarch64Only nixosModules; + }; + }; + in system: id: matrix.${system}.${id}; + in + forCheckSystems (system: let + checksForNixpkgs' = checksForNixpkgs system; + + nixpkgsChecks = checksForNixpkgs' { + input = nixpkgs; + id = "nixpkgs"; + }; + + nixosChecks = checksForNixpkgs' { + input = nixos; + id = "nixos"; + }; + in nixpkgsChecks // nixosChecks); } ); } From f82ebc21657552b2d8b5474fb2c11b65735b8c78 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Fri, 20 May 2022 13:59:29 -0400 Subject: [PATCH 13/14] Remove GitLab CI and Travis CI configs --- .gitlab-ci.yml | 132 ------------------------------------------------- .included.yml | 32 ------------ .travis.yml | 6 --- 3 files changed, 170 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100644 .included.yml delete mode 100644 .travis.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 54162653..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,132 +0,0 @@ -include: .included.yml - -variables: - GIT_DEPTH: 1 - nixpkgs_ver: "19.09" - uploadPath: ./nixos-$CI_JOB_NAME-$CI_PIPELINE_ID - -# job names are sorted alphabetically - -azure: - variables: - formats: azure - extends: - - .template - -# doesn't build -.cloudstack: - variables: - formats: cloudstack - extends: - - .template - -do: - variables: - nixpkgs_ver: unstable - formats: do - extends: - - .template - -gce: - variables: - formats: gce - extends: - - .template - -install-iso: - variables: - formats: install-iso - extends: - - .template - -install-iso-hyperv: - variables: - formats: install-iso-hyperv - extends: - - .template - -iso: - variables: - formats: iso - extends: - - .template - -kexec: - variables: - formats: kexec - extends: - - .template - -kexec-bundle: - variables: - formats: kexec-bundle - extends: - - .template - -lxc: - variables: - formats: lxc - extends: - - .template - -lxc-metadata: - variables: - formats: lxc-metadata - extends: - - .template - -openstack: - variables: - formats: openstack - extends: - - .template - -qcow: - variables: - formats: qcow - extends: - - .template - -raw: - variables: - formats: raw - extends: - - .template - -# todo -.sd-aarch64: - variables: - formats: sd-aarch64 - script: - - nix-shell --command './nixos-generate -f $formats --nixpkgs_ver $nixpkgs_ver --system aarch64-linux' - extends: - - .template - -# todo -.sd-aarch64-installer: - variables: - formats: sd-aarch64-installer - script: - - nix-shell --command './nixos-generate -f $formats --nixpkgs_ver $nixpkgs_ver --system aarch64-linux' - extends: - - .template - -virtualbox: - variables: - formats: virtualbox - extends: - - .template - -vm: - variables: - formats: vm - extends: - - .template - - .tar_option - -vm-nogui: - variables: - formats: vm-nogui - extends: - - .template - - .tar_option diff --git a/.included.yml b/.included.yml deleted file mode 100644 index e7926c55..00000000 --- a/.included.yml +++ /dev/null @@ -1,32 +0,0 @@ -stages: - - build - -.tar_option: - after_script: - - tar -cvhf $uploadPath.tar ./$uploadPath - artifacts: - name: "$uploadPath" - paths: - - $uploadPath.tar - -.template: - only: - - web - stage: build - image: lnl7/nix:latest - before_script: - - nix --version - - chmod +w /etc/nix/nix.conf - - echo "system-features = kvm" >> /etc/nix/nix.conf - - sed -i -e 's/ --no-out-link//g' nixos-generate - - mkdir $uploadPath - script: - - nix-shell --command './nixos-generate -f $formats -I nixpkgs=channel:nixos-$nixpkgs_ver' |& tee $uploadPath/build.log - # Last line of output should be the path of a regular file (or symlink - # to a regular file) in the Nix store. - - storePath=$(tail -n 1 $uploadPath/build.log) && test -f "$storePath" && test "/nix/store/${storePath#/nix/store/}" = "$storePath" - - mv $(realpath result) $uploadPath - artifacts: - name: "$uploadPath" - paths: - - $uploadPath diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4da991be..00000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: nix - -sudo: false - -script: - - nix-build From a792c3268a06bc997e893648d78d9755c7ee1ac6 Mon Sep 17 00:00:00 2001 From: Matt Schreiber Date: Mon, 16 May 2022 08:54:50 -0400 Subject: [PATCH 14/14] GitHub Actions: add image-building jobs --- .github/actions/generate-format/action.yml | 37 ++++++ .../generate-format/generate-format.sh | 74 +++++++++++ .../actions/get-flake-input-rev/action.yml | 27 ++++ .../get-flake-input-rev.sh | 18 +++ .github/actions/list-formats/action.yml | 27 ++++ .github/actions/list-formats/list-formats.sh | 12 ++ .github/workflows/build.yml | 121 +++++++++++++++++- 7 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 .github/actions/generate-format/action.yml create mode 100755 .github/actions/generate-format/generate-format.sh create mode 100644 .github/actions/get-flake-input-rev/action.yml create mode 100755 .github/actions/get-flake-input-rev/get-flake-input-rev.sh create mode 100644 .github/actions/list-formats/action.yml create mode 100755 .github/actions/list-formats/list-formats.sh diff --git a/.github/actions/generate-format/action.yml b/.github/actions/generate-format/action.yml new file mode 100644 index 00000000..46f4b43c --- /dev/null +++ b/.github/actions/generate-format/action.yml @@ -0,0 +1,37 @@ +name: Generate Format +description: Build a NixOS configuration using the specified nixos-generators format +inputs: + nix-exec-path: + description: Path to the nix executable + default: nix + format: + description: nixos-generate format name + system: + description: Target nixpkgs system (e.g. x86_64-linux) + default: x86_64-linux + nixpkgs-name: + description: Flake input name for the target nixpkgs version + default: nixpkgs + nixpkgs: + description: Search path/URL for the target nixpkgs + default: 'https://github.com/nixos/nixpkgs/archive/nixpkgs-unstable.tar.gz' + force-build: + description: Build the target format even if no corresponding flake check exists + flake: + description: Nix flake URL + default: ${{ github.workspace }} +runs: + using: composite + steps: + - name: Build the "${{ inputs.format }}" format + id: generate + shell: bash + run: ${{ github.action_path }}/generate-format.sh + env: + NIX_EXEC_PATH: ${{ inputs.nix-exec-path }} + FORMAT: ${{ inputs.format }} + SYSTEM: ${{ inputs.system }} + NIXPKGS_NAME: ${{ inputs.nixpkgs-name }} + NIXPKGS: ${{ inputs.nixpkgs }} + FORCE_BUILD: ${{ inputs.force-build }} + FLAKE: ${{ inputs.flake }} diff --git a/.github/actions/generate-format/generate-format.sh b/.github/actions/generate-format/generate-format.sh new file mode 100755 index 00000000..62a1268e --- /dev/null +++ b/.github/actions/generate-format/generate-format.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +set -x + +printenv + +die() { + rc="$?" + echo "::error file=nixos-generate::$*" + exit "$rc" +} + +generate() { + timeout 20m \ + "${NIX_EXEC_PATH:-nix}" run . \ + -- \ + -I "nixpkgs=${NIXPKGS?}" \ + "$@" +} + +getCheck() { + "${NIX_EXEC_PATH:-nix}" eval --json "${FLAKE:-.}#checks.\"${1?}\"" --apply "(builtins.hasAttr \"${2?}\")" +} + +hasCheck() { + has_check=$(getCheck "$@") || die "failed to confirm availablity of check output" + [ "$has_check" = true ] +} + +buildCheck() { + "${NIX_EXEC_PATH:-nix}" build "${FLAKE:-.}#checks.\"${1?}\".\"${2?}\"" +} + +buildAnyway() { + [ -n "${FORCE_BUILD:-}" ] +} + +checkOutputs() { + path_var="$1" + shift + + path="$1" + shift + + test_type="$1" + shift + + test "$test_type" "$path" || die "path $path does not exist or is not the expected type" + real=$(readlink -f "$path") || die "unable to resolve path to $path" + store_paths=$(nix-store -q --outputs "$real") || die "unable to get store path of $real" + echo "::set-output name=${path_var}::$(echo "$store_paths" | head -n 1)" +} + +format="${FORMAT?}" +system="${SYSTEM:-x86_64-linux}" +nixpkgs_name="${NIXPKGS_NAME:-nixpkgs}" +check="${format}-${nixpkgs_name}" +out_link="./result-${format}" + +if hasCheck "$system" "$check"; then + : # NOP +elif buildAnyway; then + buildCheck() { : ; } +else + printf 1>&2 -- "No flake check defined for format '%s' on system '%s' using nixpkgs '%s', and force-building is not enabled; exiting.\\n" \ + "$format" "$system" "$nixpkgs_name" + + exit 0 +fi + +out=$(generate -f "$format" --system "$system" -o "$out_link") || die "build exited with status $?" +buildCheck "$system" "$check" || die "flake build exited with status $?" +checkOutputs out "$out" -f +checkOutputs out_link "$out_link" -e diff --git a/.github/actions/get-flake-input-rev/action.yml b/.github/actions/get-flake-input-rev/action.yml new file mode 100644 index 00000000..0553c92a --- /dev/null +++ b/.github/actions/get-flake-input-rev/action.yml @@ -0,0 +1,27 @@ +name: Get Rev +description: Get a Nix flake input revision from the flake lock file +inputs: + nix-exec-path: + description: Path to the nix executable + default: nix + flake-input-name: + description: Name of the flake input whose revision should be retrieved + default: nixpkgs + flake: + description: Nix flake URL + default: ${{ github.workspace }} +outputs: + rev: + description: Locked revision of the flake input + value: ${{ steps.get-rev.outputs.rev }} +runs: + using: composite + steps: + - name: Get flake input revision for "${{ inputs.flake-input-name }}" from ${{ inputs.flake-lock-path }} + id: get-rev + shell: bash + env: + NIX_EXEC_PATH: ${{ inputs.nix-exec-path }} + FLAKE_INPUT_NAME: ${{ inputs.flake-input-name }} + FLAKE: ${{ inputs.flake }} + run: ${{ github.action_path }}/get-flake-input-rev.sh diff --git a/.github/actions/get-flake-input-rev/get-flake-input-rev.sh b/.github/actions/get-flake-input-rev/get-flake-input-rev.sh new file mode 100755 index 00000000..89d02d66 --- /dev/null +++ b/.github/actions/get-flake-input-rev/get-flake-input-rev.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +getRev() { + # Run in nix shell in order to use jq + # shellcheck disable=SC2016 + "${NIX_EXEC_PATH:-nix}" develop "${FLAKE:-.}" --command \ + bash -c '"$1" flake metadata --json "$2" | jq -r --arg input "$3" ".locks.nodes[\$input].locked.rev"' \ + "$0" "${NIX_EXEC_PATH:-nix}" "${FLAKE:-.}" "${1?}" +} + +rev=$(getRev "${FLAKE_INPUT_NAME:-nixpkgs}") || exit + +if [ "$rev" = null ]; then + echo "::error file=${FLAKE:-.}::unable to retrieve revision for flake input ${FLAKE_INPUT_NAME:-nixpkgs}" + exit 1 +fi + +echo "::set-output name=rev::${rev}" diff --git a/.github/actions/list-formats/action.yml b/.github/actions/list-formats/action.yml new file mode 100644 index 00000000..66049829 --- /dev/null +++ b/.github/actions/list-formats/action.yml @@ -0,0 +1,27 @@ +name: List Formats +description: List available nixos-generators formats +inputs: + nixos-generate-exec-path: + description: Path to the nixos-generate executable + default: ${{ github.workspace }}/nixos-generate + nix-exec-path: + description: Path to the nix executable + default: nix + flake: + description: Nix flake URL + default: ${{ github.workspace }} +outputs: + formats: + description: Available nixos-generators formats + value: ${{ steps.list-formats.outputs.formats }} +runs: + using: composite + steps: + - name: List available nixos-generators formats + id: list-formats + shell: bash + env: + NIXOS_GENERATE_EXEC_PATH: ${{ inputs.nixos-generate-exec-path }} + NIX_EXEC_PATH: ${{ inputs.nix-exec-path }} + FLAKE: ${{ inputs.flake }} + run: ${{ github.action_path }}/list-formats.sh diff --git a/.github/actions/list-formats/list-formats.sh b/.github/actions/list-formats/list-formats.sh new file mode 100755 index 00000000..c125c04e --- /dev/null +++ b/.github/actions/list-formats/list-formats.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +listFormats() { + # Run in nix shell in order to use jq + # shellcheck disable=SC2016 + "${NIX_EXEC_PATH:-nix}" develop "${FLAKE:-.}" --command \ + bash -c '"$1" --list | jq -cnMR "[inputs]"' \ + "$0" "${NIXOS_GENERATE_EXEC_PATH:-./nixos-generate}" +} + +formats=$(listFormats) || exit +echo "::set-output name=formats::${formats}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca373b9e..e9d1ad4c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,10 +11,127 @@ jobs: steps: - uses: actions/checkout@v3 with: - # Nix Flakes doesn't work on shallow clones - fetch-depth: 0 + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 - uses: cachix/install-nix-action@v17 - name: List flake structure run: nix flake show - name: Run unit tests (flake) run: nix build -L + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v17 + - name: Run flake checks + run: nix flake check --no-build --keep-going + introspect: + runs-on: ubuntu-latest + outputs: + nixpkgs-rev: ${{ steps.nixpkgs-rev.outputs.rev }} + nixos-rev: ${{ steps.nixos-rev.outputs.rev }} + formats: ${{ steps.list-formats.outputs.formats }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v17 + - name: Get nixpkgs flake input rev + id: nixpkgs-rev + uses: ./.github/actions/get-flake-input-rev + with: + flake-input-name: nixpkgs + - name: Get nixos flake input rev + id: nixos-rev + uses: ./.github/actions/get-flake-input-rev + with: + flake-input-name: nixos + - name: List available formats + id: list-formats + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + uses: ./.github/actions/list-formats + default-formats: + strategy: + matrix: + format: + - amazon + - docker + - iso + - kexec-bundle + nixpkgs-name: + - nixpkgs + - nixos + include: + - nixpkgs-name: nixpkgs + nixpkgs: 'https://github.com/nixos/nixpkgs/archive/${{ needs.introspect.outputs.nixpkgs-rev }}.tar.gz' + - nixpkgs-name: nixos + nixpkgs: 'https://github.com/nixos/nixpkgs/archive/${{ needs.introspect.outputs.nixos-rev }}.tar.gz' + - format: kexec-bundle + force-build: yes + runs-on: ubuntu-latest + needs: [ 'check', 'introspect' ] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v17 + with: + # kvm required for a number of formats; big-parallel required for + # proxmox and possibly others. + extra_nix_config: | + system-features = big-parallel kvm + - name: Build the "${{ matrix.format }}" format + id: generate + uses: ./.github/actions/generate-format + with: + format: ${{ matrix.format }} + system: ${{ matrix.system }} + nixpkgs-name: ${{ matrix.nixpkgs-name }} + nixpkgs: ${{ matrix.nixpkgs }} + force-build: ${{ matrix.force-build }} + all-formats: + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + strategy: + matrix: + format: ${{ needs.introspect.outputs.formats && fromJSON(needs.introspect.outputs.formats) }} + nixpkgs-name: + - nixpkgs + - nixos + include: + - nixpkgs-name: nixpkgs + nixpkgs: 'https://github.com/nixos/nixpkgs/archive/${{ needs.introspect.outputs.nixpkgs-rev }}.tar.gz' + - nixpkgs-name: nixos + nixpkgs: 'https://github.com/nixos/nixpkgs/archive/${{ needs.introspect.outputs.nixos-rev }}.tar.gz' + - format: kexec-bundle + force-build: yes + - format: sd-aarch64-installer + system: aarch64-linux + - format: sd-aarch64 + system: aarch64-linux + runs-on: ubuntu-latest + needs: [ 'check', 'introspect' ] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + # set up qemu if we are targeting a non-native system + - uses: docker/setup-qemu-action@v2 + if: ${{ matrix.system }} + - uses: cachix/install-nix-action@v17 + with: + # kvm required for a number of formats; big-parallel required for + # proxmox and possibly others. + extra_nix_config: | + system-features = big-parallel kvm + extra-platforms = ${{ matrix.system }} + - name: Build the "${{ matrix.format }}" format + id: generate + uses: ./.github/actions/generate-format + with: + format: ${{ matrix.format }} + system: ${{ matrix.system }} + nixpkgs-name: ${{ matrix.nixpkgs-name }} + nixpkgs: ${{ matrix.nixpkgs }} + force-build: ${{ matrix.force-build }}