diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 60e51cf..4c8a00f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -48,17 +48,4 @@ steps: # Stack via stack2nix - label: static-stack - # TODO: Remove the override of unix-compat below once that's available in stack - command: | - set -eu -o pipefail - - cd static-stack/ - mkdir -p static-stack-test-dir - curl -L https://github.com/commercialhaskell/stack/archive/v2.1.3.tar.gz | tar -xz -C static-stack-test-dir - # Use lts-12 because ghc822 is no longer in nixpkgs - cp static-stack-test-dir/stack-*/stack-lts-12.yaml static-stack-test-dir/stack-*/stack.yaml - - echo "Overriding point release of unix-compat, see https://github.com/nh2/static-haskell-nix/issues/79" - perl -pi -e 's/^packages:/packages:\n- unix-compat-0.5.2\@rev:0/g' static-stack-test-dir/stack-*/snapshot-lts-12.yaml - - $(nix-build --no-link -A fullBuildScript --argstr stackDir $PWD/static-stack-test-dir/stack-*) + command: "cd static-stack && ./build-static-stack.sh" diff --git a/README.md b/README.md index 131e535..95b1000 100644 --- a/README.md +++ b/README.md @@ -116,5 +116,7 @@ The [`static-stack`](./static-stack) directory shows how Stack itself can be bui then `/nix/store/dax3wjbjfrcwj6r3mafxj5fx6wcg5zbp-stack-2.3.0.1` is your final output _store path_ whose `/bin` directory contains your static executable. * I get `stack2nix: user error (No such package mypackage-1.2.3 in the cabal database. Did you run cabal update?)`. * You most likely have to bump the date like `hackageSnapshot = "2019-05-08T00:00:00Z";` to a newer date (past the time that package-version was added to Hackage). +* Can I build Stack projects with resolvers that are too old to be supported by Stack >= 2? + * No. For that you need need to use an old `static-haskell-nix` version: The one before [this PR](https://github.com/nh2/static-haskell-nix/pull/98) was merged. * I get some other error. Can I just file an issue and have you help me with it? * Yes. If possible (especially if your project is open source), please push some code so that your issue can be easily reproduced. diff --git a/nixpkgs b/nixpkgs index 0c96026..8d536f3 160000 --- a/nixpkgs +++ b/nixpkgs @@ -1 +1 @@ -Subproject commit 0c960262d159d3a884dadc3d4e4b131557dad116 +Subproject commit 8d536f36256d30d8fa47b24caafb1af6405889f3 diff --git a/nixpkgs.nix b/nixpkgs.nix index 99524db..ac48a41 100644 --- a/nixpkgs.nix +++ b/nixpkgs.nix @@ -25,4 +25,4 @@ if builtins.getEnv "STATIC_HASKELL_NIX_CI_NIXPKGS_UNSTABLE_BUILD" == "1" if builtins.pathExists ./nixpkgs/pkgs then import ./nixpkgs {} # Pinned nixpkgs version; should be kept up-to-date with our submodule. - else import (fetchTarball https://github.com/NixOS/nixpkgs/archive/0c960262d159d3a884dadc3d4e4b131557dad116.tar.gz) {} + else import (fetchTarball https://github.com/nh2/nixpkgs/archive/8d536f36256d30d8fa47b24caafb1af6405889f3.tar.gz) {} diff --git a/static-stack/README.md b/static-stack/README.md index 6dbfd5b..fa7ae99 100644 --- a/static-stack/README.md +++ b/static-stack/README.md @@ -18,16 +18,6 @@ If you get an error such as: then update the `hackageSnapshot` date in `default.nix` to a date that includes the package and version. -### Temporarily: Use `stack-lts-12.yaml` - -See #81: Stack's main `stack.yaml` uses GHC 8.2.2, which is no longer in recent `nixpkgs`. - -If you get an error mentioning `error: attribute 'ghc822' in selection path 'haskell.compiler.ghc822' not found`, then build stack the same way [as our CI does it](https://github.com/nh2/static-haskell-nix/blob/38ef5a4e00b5f6fb421014829320d85899483874/.buildkite/pipeline.yml#L50-L64). - -Alternatively you can use an older version of `static-haskell-nix` that pins an older version of `nixpkgs` that still has `ghc822`. - -This issue will likely go away very soon given that [stack is as-of-writing upgrading `stack.yaml` to a newer LTS](https://github.com/commercialhaskell/stack/pull/5162). - ## Binary caches for faster building (optional) You can use the caches described in the [top-level README](../README.md#binary-caches-for-faster-building-optional) for faster building. diff --git a/static-stack/build-static-stack.sh b/static-stack/build-static-stack.sh new file mode 100755 index 0000000..929c4a2 --- /dev/null +++ b/static-stack/build-static-stack.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash + +set -eu -o pipefail + +mkdir -p static-stack-test-dir +curl -L https://github.com/commercialhaskell/stack/archive/v2.7.1.tar.gz | tar -xz -C static-stack-test-dir + +$(nix-build --no-link -A fullBuildScript --argstr stackDir $PWD/static-stack-test-dir/stack-*) diff --git a/static-stack/default.nix b/static-stack/default.nix index 082d11b..c3df645 100644 --- a/static-stack/default.nix +++ b/static-stack/default.nix @@ -9,14 +9,15 @@ }: let cabalPackageName = "stack"; - compiler = "ghc844"; # matching stack-lts-12.yaml + compiler = "ghc8104"; # matching stack-lts-12.yaml pkgs = import ../nixpkgs {}; stack2nix-script = import ../static-stack2nix-builder/stack2nix-script.nix { - pkgs = pkgs; + inherit pkgs; + inherit compiler; stack-project-dir = stackDir; # where stack.yaml is - hackageSnapshot = "2020-02-08T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions + hackageSnapshot = "2021-07-12T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions }; static-stack2nix-builder = import ../static-stack2nix-builder/default.nix { diff --git a/static-stack2nix-builder-example/default.nix b/static-stack2nix-builder-example/default.nix index dfce56b..e3490f7 100644 --- a/static-stack2nix-builder-example/default.nix +++ b/static-stack2nix-builder-example/default.nix @@ -6,14 +6,14 @@ }: let cabalPackageName = "example-project"; - compiler = "ghc865"; # matching stack.yaml + compiler = "ghc8104"; # matching stack.yaml # Pin static-haskell-nix version. static-haskell-nix = if builtins.pathExists ../.in-static-haskell-nix then toString ../. # for the case that we're in static-haskell-nix itself, so that CI always builds the latest version. # Update this hash to use a different `static-haskell-nix` version: - else fetchTarball https://github.com/nh2/static-haskell-nix/archive/d1b20f35ec7d3761e59bd323bbe0cca23b3dfc82.tar.gz; + else fetchTarball https://github.com/nh2/static-haskell-nix/archive/39f2f38ed042e84d8b6627a236c1815f48023d3c.tar.gz; # Pin nixpkgs version # By default to the one `static-haskell-nix` provides, but you may also give @@ -23,8 +23,9 @@ let stack2nix-script = import "${static-haskell-nix}/static-stack2nix-builder/stack2nix-script.nix" { inherit pkgs; + inherit compiler; stack-project-dir = toString ./.; # where stack.yaml is - hackageSnapshot = "2019-10-08T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions + hackageSnapshot = "2021-07-11T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions }; static-stack2nix-builder = import "${static-haskell-nix}/static-stack2nix-builder/default.nix" { diff --git a/static-stack2nix-builder-example/stack.yaml b/static-stack2nix-builder-example/stack.yaml index c113006..e4340f1 100644 --- a/static-stack2nix-builder-example/stack.yaml +++ b/static-stack2nix-builder-example/stack.yaml @@ -1,3 +1,3 @@ -resolver: lts-14.7 +resolver: lts-18.2 packages: - . diff --git a/static-stack2nix-builder/default.nix b/static-stack2nix-builder/default.nix index d934557..b11300a 100644 --- a/static-stack2nix-builder/default.nix +++ b/static-stack2nix-builder/default.nix @@ -5,19 +5,19 @@ # The name of the cabal package to build, e.g. "pandoc". cabalPackageName ? "myproject", - # Compiler name in nixpkgs, e.g. "ghc864". + # Compiler name in nixpkgs, e.g. "ghc8104". # Must match the one in the `resolver` in `stack.yaml`. # If you get this wrong, you'll likely get an error like # : cannot satisfy -package-id Cabal-2.4.1.0-ALhzvdqe44A7vLWPOxSupv # TODO: Make `stack2nix` tell us that. - compiler ? "ghc864", + compiler ? "ghc8104", # Path to `stack2nix` output that shall be used as Haskell packages. # You should usually give this the store path that `stack2nix-script` outputs. stack2nix-output-path, # Pin nixpkgs version. - normalPkgs ? import (fetchTarball https://github.com/NixOS/nixpkgs/archive/88ae8f7d55efa457c95187011eb410d097108445.tar.gz) {}, + normalPkgs ? import (fetchTarball https://github.com/nh2/nixpkgs/archive/8d536f36256d30d8fa47b24caafb1af6405889f3.tar.gz) {}, # Use `integer-simple` instead of `integer-gmp` to avoid linking in # this LGPL dependency statically. diff --git a/static-stack2nix-builder/stack2nix-script.nix b/static-stack2nix-builder/stack2nix-script.nix index 1d68b63..41893e0 100644 --- a/static-stack2nix-builder/stack2nix-script.nix +++ b/static-stack2nix-builder/stack2nix-script.nix @@ -7,6 +7,18 @@ # nixpkgs to use. pkgs, + # ghc to use; only because without a GHC on path, stack complains: + # stack2nix: No compiler found, expected minor version match with ghc-8.10.4 (x86_64) (based on resolver setting ... + # This happens even when using the Stack API (as stack2nix does), + # and stack2nix doen't currently accept or set the `--system-ghc` + # flag to skip the check (maybe it should to eschew this option; + # I suspect our operation here never uses GHC). + # TODO: This shouldn't be necessary since `stack2nix` commit + # Set `--system-ghc` via stack API. + # But somehow stack2nix still complains about it; + # perhaps we didn't use the Stack API correctly. + compiler, + # Path to directory containing `stack.yaml`. stack-project-dir, @@ -47,37 +59,16 @@ "${pkgs.cabal-install}/bin" # `cabal` "${pkgs.nix}/bin" # various `nix-*` commands "${pkgs.wget}/bin" # `wget` + "${pkgs.haskell.compiler.${compiler}}/bin" # `ghc` version matching target stack.yaml ]; fixed_stack2nix = - let - # stack2nix isn't compatible with Stack >= 2.0, see - # https://github.com/input-output-hk/stack2nix/issues/168. - # Current versions of nixpkgs master have Stack >= 2.0, see - # https://github.com/NixOS/nixpkgs/issues/63691. - # We thus fetch the `stack2nix` binary from an older nixpkgs version - # that doesn't have Stack >= 2.0. - # This means that `static-stack2nix-builder` may not work on `stack.yaml` - # files that aren't compatible with Stack < 2.0. - stack2nix_pkgs = import (fetchTarball https://github.com/NixOS/nixpkgs/archive/e36f91fa86109fa93cac2516a9365af57233a3a6.tar.gz) {}; - in - # Some older stack2nix versions have fundamental problems that prevent - # stack2nix from running correctly. Fix them here, until these old versions - # are faded out of current nixpkgs. Especially: - # * "Make sure output is written in UTF-8." - # https://github.com/input-output-hk/stack2nix/commit/cb05818ef8b58899f15641f50cb04e5473b4f9b0 - # * "Make GHC base libraries dependent on GHC version." - # https://github.com/input-output-hk/stack2nix/pull/172/commits - # - # Versions < 0.2.4 aren't supported, force-upgrade them to 0.2.4. - if stack2nix_pkgs.lib.versionOlder stack2nix_pkgs.stack2nix.version "0.2.4" - then stack2nix_pkgs.haskellPackages.callCabal2nix "stack2nix" (stack2nix_pkgs.fetchFromGitHub { - owner = "nh2"; - repo = "stack2nix"; - rev = "c009e33af30c76b8fe94388382d816079fb5ac4e"; - sha256 = "0x0hjzjlx1a0pyjd8aalk3ajwcymsb2qd65n2sqdhpy9bdsz8vxl"; - }) {} - else stack2nix_pkgs.stack2nix; + pkgs.haskellPackages.callCabal2nix "stack2nix" (pkgs.fetchFromGitHub { + owner = "nh2"; + repo = "stack2nix"; + rev = "c20097d4edf82256484a733544579d4b5e0f2808"; + sha256 = "1lpwc20q62z9a9fpksd9q10x1jz8l29psx4dqsff759srj4chy9p"; + }) {}; in pkgs.writeShellScript "stack2nix-build-script.sh" '' set -eu -o pipefail diff --git a/survey/default.nix b/survey/default.nix index 828beaa..abd40b9 100644 --- a/survey/default.nix +++ b/survey/default.nix @@ -35,16 +35,23 @@ in ])."${approach}", # When changing this, also change the default version of Cabal declared below - compiler ? "ghc865", + compiler ? "ghc8104", + # See: + # * https://www.snoyman.com/base/ + # * https://www.haskell.org/cabal/download.html + # section "Older Releases" defaultCabalPackageVersionComingWithGhc ? ({ ghc822 = "Cabal_2_2_0_1"; # TODO this is technically incorrect for ghc 8.2.2, should be 2.0.1.0, but nixpkgs doesn't have that ghc844 = "Cabal_2_2_0_1"; ghc863 = throw "static-haskell-nix: ghc863 is no longer supported, please upgrade"; - ghc864 = throw "static-haskell-nix: ghc863 is no longer supported, please upgrade"; + ghc864 = throw "static-haskell-nix: ghc864 is no longer supported, please upgrade"; ghc865 = "Cabal_2_4_1_0"; # TODO this is technically incorrect for ghc 8.6.5, should be 2.4.0.1, but nixpkgs doesn't have that ghc881 = "Cabal_3_0_0_0"; + ghc8104 = "Cabal_3_2_1_0"; + ghc8105 = "Cabal_3_2_1_0"; + ghc901 = "Cabal_3_4_0_0"; }."${compiler}"), # Use `integer-simple` instead of `integer-gmp` to avoid linking in @@ -96,27 +103,17 @@ let passthru.isExecutable = drv.isExecutable or false; })).isExecutable; - # Function that tells us if a given Haskell package is marked as broken. - isBroken = pkg: pkg.meta.broken or false; + # Turn e.g. `Cabal_1_2_3_4` into `1.2.3.4`. + cabalDottedVersion = + builtins.replaceStrings ["_"] ["."] + (builtins.substring + (builtins.stringLength "Cabal_") + (builtins.stringLength defaultCabalPackageVersionComingWithGhc) + defaultCabalPackageVersionComingWithGhc + ); - # Function that for a given Haskell package tells us if any of - # its dependencies is marked as `broken`. - hasBrokenDeps = pkg: - let - libraryDepends = - (pkgs.haskell.lib.overrideCabal pkg (drv: { - passthru.libraryHaskellDepends = drv.libraryHaskellDepends or []; - })).libraryHaskellDepends; - in - lib.any (x: - let - res = builtins.tryEval (isProperHaskellPackage x && isBroken x); - broken = res.success && res.value; - in - if broken - then trace "broken because of broken deps: ${pkg}" broken - else broken - ) libraryDepends; + areCabalPatchesRequired = + builtins.length (requiredCabalPatchesList cabalDottedVersion) != 0; # Nixpkgs contains both Hackage and Stackage packages. # We want to build only executables that are on Stackage because @@ -128,14 +125,14 @@ let # Contains a list of package names (strings). stackagePackages = let - stackageInfoPath = pkgs.path + "/pkgs/development/haskell-modules/configuration-hackage2nix.yaml"; + stackageInfoPath = pkgs.path + "/pkgs/development/haskell-modules/configuration-hackage2nix/stackage.yaml"; pythonWithYaml = pkgs.python2Packages.python.withPackages (pkgs: [pkgs.pyyaml]); - dont-distribute-packages-file = normalPkgs.runCommand "test" {} '' + stackage-packages-file = normalPkgs.runCommand "stackage-packages" {} '' ${pythonWithYaml}/bin/python -c 'import yaml, json; x = yaml.load(open("${stackageInfoPath}")); print(json.dumps([line.split(" ")[0] for line in x["default-package-overrides"]]))' > $out ''; - dont-distribute-packages = builtins.fromJSON (builtins.readFile dont-distribute-packages-file); + stackage-packages = builtins.fromJSON (builtins.readFile stackage-packages-file); in - dont-distribute-packages; + stackage-packages; # Turns a list into a "set" (map where all values are {}). keySet = list: builtins.listToAttrs (map (name: lib.nameValuePair name {}) list); @@ -192,10 +189,18 @@ let # The best choice for that is the version that comes with the GHC. # Unfortunately we can't query that easily, so we maintain that manually # in `defaultCabalPackageVersionComingWithGhc`. - # That effort will go away once all our Cabal patches are upstreamed. - if builtins.isNull super.Cabal - then applyPatchesToCabalDrv super."${defaultCabalPackageVersionComingWithGhc}" - else applyPatchesToCabalDrv super.Cabal; + # Currently all our Cabal patches are upstreamed, so this is technically + # not necessary currently; however, we keep this infrastructure in case + # we need to patch Cabal again in the future. + # + # If there are no patches to apply, keep original Cabal, + # even if `null` (to get the one that comes with GHC). + if not areCabalPatchesRequired + then super.Cabal + else + if builtins.isNull super.Cabal + then applyPatchesToCabalDrv super."${defaultCabalPackageVersionComingWithGhc}" + else applyPatchesToCabalDrv super.Cabal; }); }); @@ -254,10 +259,38 @@ let # Stackage package names we want to blacklist. blacklist = [ + # TODO: Try to remove when https://github.com/NixOS/nixpkgs/pull/128746 is available to us + "alsa-pcm" "alsa-seq" "ALUT" "OpenAL" "sdl2" "sdl2-gfx" "sdl2-image" "sdl2-mixer" "sdl2-ttf" + + # depends on `sbv` -> `openjdk`, which pulls in a huge dependency closure + "crackNum" + + # Incorrectly depends on `ocaml`, which `pkgsMusl` cannot currently build + # (error: `ld: -r and -pie may not be used together`). + # TODO: Remove when https://github.com/NixOS/cabal2nix/pull/509 has landed. + "liquid-fixpoint" + # Doesn't build in `normalPkgs.haskellPackages` either "mercury-api" + # https://github.com/nh2/static-haskell-nix/issues/6#issuecomment-420494800 "sparkle" + + # These ones currently don't compile for not-yet-investigated reasons: + "amqp-utils" + "elynx" + "hopenpgp-tools" + "hw-eliasfano" + "hw-ip" + "leveldb-haskell" + "mpi-hs" + "mpi-hs-binary" + "mpi-hs-cereal" + "place-cursor-at" + "sandwich-webdriver" + "slynx" + "spacecookie" + "zip" ]; # All Stackage executables who (and whose dependencies) are not marked @@ -270,17 +303,20 @@ let then pkgs.haskell.packages.integer-simple."${compiler}" else pkgs.haskell.packages."${compiler}"; - stackageExecutables = lib.filterAttrs (name: x: isStackagePackage name && !(lib.elem name blacklist) && ( + stackageExecutables = let - res = builtins.tryEval ( - isProperHaskellPackage x - && isExecutable x - && !(isBroken x) - && !(hasBrokenDeps x) - ); + # Predicate copied from nixpkgs' `transitive-broken-packages.nix`: + isEvaluatingUnbroken = v: (builtins.tryEval (v.outPath or null)).success && lib.isDerivation v && !v.meta.broken; in - res.success && res.value) - ) normalHaskellPackages; + lib.filterAttrs + (name: p: + p != null && # packages that come with GHC are `null` + isStackagePackage name && + !(lib.elem name blacklist) && + isExecutable p && + isEvaluatingUnbroken p + ) + normalHaskellPackages; stackageExecutablesNames = builtins.attrNames stackageExecutables; nMany = lib.length stackageExecutablesNames; @@ -336,66 +372,70 @@ let in patchOnCabalLibraryFilesOnly; + # Returns the list of patches that a given cabal derivation needs to work well + # for static building. + requiredCabalPatchesList = cabalDottedVersionString: + # Patches we know are merged in a certain cabal version + # (we include them conditionally here anyway, for the case + # that the user specifies a different Cabal version e.g. via + # `stack2nix`): + if pkgs.lib.versionOlder cabalDottedVersionString "3.0.0.0" + then + (builtins.concatLists [ + # -L flag deduplication + # https://github.com/haskell/cabal/pull/5356 + (lib.optional (pkgs.lib.versionOlder cabalDottedVersionString "2.4.0.0") (makeCabalPatch { + name = "5356.patch"; + url = "https://github.com/haskell/cabal/commit/fd6ff29e268063f8a5135b06aed35856b87dd991.patch"; + sha256 = "1l5zwrbdrra789c2sppvdrw3b8jq241fgavb8lnvlaqq7sagzd1r"; + })) + # Patches that as of writing aren't merged yet: + ]) ++ [ + # TODO Move this into the above section when merged in some Cabal version: + # --enable-executable-static + # https://github.com/haskell/cabal/pull/5446 + (if pkgs.lib.versionOlder cabalDottedVersionString "2.4.0.0" + then + # Older cabal, from https://github.com/nh2/cabal/commits/dedupe-more-include-and-linker-flags-enable-static-executables-flag-pass-ld-options-to-ghc-Cabal-v2.2.0.1 + (makeCabalPatch { + name = "5446.patch"; + url = "https://github.com/haskell/cabal/commit/748f07b50724f2618798d200894f387020afc300.patch"; + sha256 = "1zmbalkdbd1xyf0kw5js74bpifhzhm16c98kn7kkgrwql1pbdyp5"; + }) + else + (makeCabalPatch { + name = "5446.patch"; + url = "https://github.com/haskell/cabal/commit/cb221c23c274f79dcab65aef3756377af113ae21.patch"; + sha256 = "02qalj5y35lq22f19sz3c18syym53d6bdqzbnx9f6z3m7xg591p1"; + }) + ) + # TODO Move this into the above section when merged in some Cabal version: + # ld-option passthrough + # https://github.com/haskell/cabal/pull/5451 + (if pkgs.lib.versionOlder cabalDottedVersionString "2.4.0.0" + then + # Older cabal, from https://github.com/nh2/cabal/commits/dedupe-more-include-and-linker-flags-enable-static-executables-flag-pass-ld-options-to-ghc-Cabal-v2.2.0.1 + (makeCabalPatch { + name = "5451.patch"; + url = "https://github.com/haskell/cabal/commit/b66be72db3b34ea63144b45fcaf61822e0fade87.patch"; + sha256 = "0hndkfb96ry925xzx85km8y8pfv5ka5jz3jvy3m4l23jsrsd06c9"; + }) + else + (makeCabalPatch { + name = "5451.patch"; + url = "https://github.com/haskell/cabal/commit/0aeb541393c0fce6099ea7b0366c956e18937791.patch"; + sha256 = "0pa9r79730n1kah8x54jrd6zraahri21jahasn7k4ng30rnnidgz"; + }) + ) + ] + # cabal >= 3.0.0.0 currently needs no patches. + else []; + applyPatchesToCabalDrv = cabalDrv: pkgs.haskell.lib.overrideCabal cabalDrv (old: { - patches = - # Patches we know are merged in a certain cabal version - # (we include them conditionally here anyway, for the case - # that the user specifies a different Cabal version e.g. via - # `stack2nix`): - if pkgs.lib.versionOlder cabalDrv.version "3.0.0.0" - then - (builtins.concatLists [ - # -L flag deduplication - # https://github.com/haskell/cabal/pull/5356 - (lib.optional (pkgs.lib.versionOlder cabalDrv.version "2.4.0.0") (makeCabalPatch { - name = "5356.patch"; - url = "https://github.com/haskell/cabal/commit/fd6ff29e268063f8a5135b06aed35856b87dd991.patch"; - sha256 = "1l5zwrbdrra789c2sppvdrw3b8jq241fgavb8lnvlaqq7sagzd1r"; - })) - # Patches that as of writing aren't merged yet: - ]) ++ [ - # TODO Move this into the above section when merged in some Cabal version: - # --enable-executable-static - # https://github.com/haskell/cabal/pull/5446 - (if pkgs.lib.versionOlder cabalDrv.version "2.4.0.0" - then - # Older cabal, from https://github.com/nh2/cabal/commits/dedupe-more-include-and-linker-flags-enable-static-executables-flag-pass-ld-options-to-ghc-Cabal-v2.2.0.1 - (makeCabalPatch { - name = "5446.patch"; - url = "https://github.com/haskell/cabal/commit/748f07b50724f2618798d200894f387020afc300.patch"; - sha256 = "1zmbalkdbd1xyf0kw5js74bpifhzhm16c98kn7kkgrwql1pbdyp5"; - }) - else - (makeCabalPatch { - name = "5446.patch"; - url = "https://github.com/haskell/cabal/commit/cb221c23c274f79dcab65aef3756377af113ae21.patch"; - sha256 = "02qalj5y35lq22f19sz3c18syym53d6bdqzbnx9f6z3m7xg591p1"; - }) - ) - # TODO Move this into the above section when merged in some Cabal version: - # ld-option passthrough - # https://github.com/haskell/cabal/pull/5451 - (if pkgs.lib.versionOlder cabalDrv.version "2.4.0.0" - then - # Older cabal, from https://github.com/nh2/cabal/commits/dedupe-more-include-and-linker-flags-enable-static-executables-flag-pass-ld-options-to-ghc-Cabal-v2.2.0.1 - (makeCabalPatch { - name = "5451.patch"; - url = "https://github.com/haskell/cabal/commit/b66be72db3b34ea63144b45fcaf61822e0fade87.patch"; - sha256 = "0hndkfb96ry925xzx85km8y8pfv5ka5jz3jvy3m4l23jsrsd06c9"; - }) - else - (makeCabalPatch { - name = "5451.patch"; - url = "https://github.com/haskell/cabal/commit/0aeb541393c0fce6099ea7b0366c956e18937791.patch"; - sha256 = "0pa9r79730n1kah8x54jrd6zraahri21jahasn7k4ng30rnnidgz"; - }) - ) - ] - # cabal >= 3.0.0.0 currently needs no patches. - else []; + patches = (old.patches or []) ++ requiredCabalPatchesList cabalDrv.version; }); - useFixedCabal = + useFixedCabal = if !areCabalPatchesRequired then (drv: drv) else let patchIfCabal = drv: if (drv.pname or "") == "Cabal" # the `ghc` package has not `pname` attribute, so we default to "" here @@ -481,6 +521,29 @@ let })).overrideAttrs (old: { dontDisableStatic = true; + configureFlags = (old.configureFlags or []) ++ [ + "--enable-static" + # Use environment variable to override the `pkg-config` command + # to have `--static`, as even curl's `--enable-static` configure option + # does not currently make it itself invoke `pkg-config` with that flag. + # See: https://github.com/curl/curl/issues/503#issuecomment-150680789 + # While one would usually do + # PKG_CONFIG="pkg-config --static" ./configure ... + # nix's generic stdenv builder does not support passing environment + # variables before `./configure`, and doing `PKG_CONFIG = "false";` + # as a nix attribute doesn't work either for unknown reasons + # (it gets set in the `bash` executing the build, but something resets + # it for the child process invocations); luckily, `./configure` + # also accepts env variables at the end as arguments. + # However, they apparently have to be single paths, so passing + # ./configure ... PKG_CONFIG="pkg-config --static" + # does not work, so we use `writeScript` instead. + # + # (Personally I think that passing `--enable-static` to curl should + # probably instruct it to pass `--static` to `pkg-config` itself.) + "PKG_CONFIG=${pkgs.writeScript "pkgconfig-static-wrapper" "exec pkg-config --static $@"}" + ]; + # Additionally, flags to also build a static `curl` executable: # Note: It is important that in the eventual `libtool` invocation, @@ -607,6 +670,14 @@ let # We need libpq, which does not need systemd, # and systemd doesn't currently build with musl. enableSystemd = false; + # Kerberos is problematic on static: + # configure: error: could not find function 'gss_init_sec_context' required for GSSAPI + gssSupport = false; + }; + + procps = previous.procps.override { + # systemd doesn't currently build with musl. + withSystemd = false; }; pixman = previous.pixman.overrideAttrs (old: { dontDisableStatic = true; }); @@ -617,6 +688,15 @@ let "--enable-static" ]; }); + fontforge = previous.fontforge.override ({ + # Currently produces linker errors like: + # ld: ../../lib/libfontforge.so.4: undefined reference to `_kBrotliPrefixCodeRanges' + # ld: ../../lib/libfontforge.so.4: undefined reference to `woff2::Write255UShort(std::vector + # ld: ../../lib/libfontforge.so.4: undefined reference to `BrotliGetDictionary' + # ld: ../../lib/libfontforge.so.4: undefined reference to `woff2::Store255UShort(int, unsigned long*, unsigned char*)' + # ld: ../../lib/libfontforge.so.4: undefined reference to `BrotliDefaultAllocFunc' + woff2 = null; + }); cairo = previous.cairo.overrideAttrs (old: { dontDisableStatic = true; }); libpng = previous.libpng.overrideAttrs (old: { dontDisableStatic = true; }); libpng_apng = previous.libpng_apng.overrideAttrs (old: { dontDisableStatic = true; }); @@ -682,6 +762,14 @@ let staticOnly = true; }; + # Brotli can currently build only static or shared libraries, + # see https://github.com/google/brotli/pull/655#issuecomment-864395830 + brotli = previous.brotli.override { staticOnly = true; }; + + # woff2 currently builds against the `brotli` static libs only with a patch + # that's enabled by its `static` argument. + woff2 = previous.woff2.override { static = true; }; + # See comments on `statify_curl_including_exe` for the interaction with krb5! # As mentioned in [Packages that can't be overridden by overlays], we can't # override zlib to have static libs, so we have to pass in `zlib_both` explicitly @@ -699,7 +787,9 @@ let # Can't use `zlib_both` here (infinite recursion), so we # re-`statify_zlib` `final.zlib` here (interesting that # `previous.zlib` also leads to infinite recursion at time of writing). - curl = old.curl.override { zlib = statify_zlib final.zlib; }; + # We also disable kerberos (`gssSupport`) here again, because for + # some unknown reason it sneaks back in. + curl = old.curl.override { zlib = statify_zlib final.zlib; gssSupport = false; }; }); R = (previous.R.override { @@ -715,6 +805,46 @@ let # https://git.alpinelinux.org/aports/tree/community/R/APKBUILD?id=e2bce14c748aacb867713cb81a91fad6e8e7f7f6#n56 doCheck = false; }); + + bash-completion = previous.bash-completion.overrideAttrs (old: { + # Disable tests because it some of them seem dynamic linking specific: + # FAILED test/t/test_getconf.py::TestGetconf::test_1 - assert + doCheck = false; + }); + + # As of writing, emacs still doesn't build, erroring with: + # Segmentation fault ./temacs --batch --no-build-details --load loadup bootstrap + emacs = previous.emacs.override { + # Requires librsvg (in Rust), which gives: + # missing bootstrap url for platform x86_64-unknown-linux-musl + withX = false; + withGTK3 = false; # needs to be disabled because `withX` is disabled above + systemd = null; # does not build with musl + }; + + # Disable sphinx for some python packages to not pull in its huge + # dependency tree; it pulls in among others: + # Ruby, Python, Perl, Rust, OpenGL, Xorg, gtk, LLVM. + # + # Which Python packages to override is informed by this dependency tree: + # nix-store -q --tree $(NIX_PATH=nixpkgs=nixpkgs nix-instantiate survey/default.nix -A workingStackageExecutables) + # and searching for `-sphinx`. + python3 = previous.python3.override { + # Careful, we're using a different self and super here! + packageOverrides = finalPython: previousPython: { + + fonttools = previousPython.fonttools.override { + sphinx = null; + }; + + matplotlib = previousPython.matplotlib.override { + sphinx = null; + }; + + }; + }; + }; @@ -747,6 +877,8 @@ let # and GHC inserts these flags too early, that is in our case, before # the `-lcurl` that pulls in these dependencies; see # https://github.com/haskell/cabal/pull/5451#issuecomment-406759839 + # Note that current binutils emit a warning: + # ld: missing --end-group; added as last command line option # TODO: This can be removed once we have GHC 8.10, due to my merged PR: # https://gitlab.haskell.org/ghc/ghc/merge_requests/1589 "--ld-option=-Wl,--start-group" @@ -825,8 +957,36 @@ let # We use `buildFlags` instead of `configureFlags` so that it's # also in effect for packages which specify e.g. # `ghc-options: -O2` in their .cabal file. - buildFlags = (attrs.buildFlags or []) ++ - final.lib.optional disableOptimization "--ghc-option=-O0"; + buildFlags = (attrs.buildFlags or []) ++ builtins.concatLists [ + (final.lib.optional disableOptimization "--ghc-option=-O0") + # GHC compilation does not scale well on high-core machines like our CI. + # Making compilation more efficient. + # (map (o: "--ghc-option=" + o) [ + # "-j4" # Limit parallel compilation + # "+RTS" + # "-maxN4" # Limit threads + # "-A64M" "-qb0" # See https://trofi.github.io/posts/193-scaling-ghc-make.html + # "-RTS" + # ]) + ]; + + # Note [Slow stripping]: + # There is currently a 300x `strip` performance regression in + # `binutils`, making some strips take 5 minutes instead of 1 second. + # Disable stripping of libraries (`.a` files) until it's solved: + # https://github.com/NixOS/nixpkgs/issues/129467 + # https://sourceware.org/bugzilla/show_bug.cgi?id=28058 + # We continue to strip executables because they don't seem to + # be affected by the regression. + # + # There is also the consideration to keep `dontStrip = true;` the default, + # so that people can decide at the very end whether they prefer small + # executable sizes, or increased debuggability by keeping debug symbols. + # However, until the `-g` issue + # https://gitlab.haskell.org/ghc/ghc/-/issues/15960 + # is figured out, it may be best to not enable `-g`, and thus + # not-stripping isn't as useful. + configureFlags = (attrs.configureFlags or []) ++ [ "--disable-library-stripping" ]; }); # Note: @@ -846,6 +1006,16 @@ let # see note [Packages that can't be overridden by overlays]. zlib = super.zlib.override { zlib = final.zlib_both; }; + # The `properties` test suite takes > 30 minutes with `-O0`. + aeson-diff = + (if disableOptimization then dontCheck else lib.id) + super.aeson-diff; + + # Tests use `inspection-testing` which cannot work on `-O0`. + ap-normalize = + (if disableOptimization then dontCheck else lib.id) + super.ap-normalize; + # `criterion`'s test suite fails with a timeout if its dependent # libraries (apparently `bytestring`) are compiled with `-O0`. # Even increasing the timeout 5x did not help! @@ -888,6 +1058,11 @@ let (if disableOptimization then dontCheck else lib.id) super.weigh; + # Tests use `inspection-testing` which cannot work on `-O0`. + generic-data = + (if disableOptimization then dontCheck else lib.id) + super.generic-data; + # `HsOpenSSL` has a bug where assertions are only triggered on `-O0`. # This breaks its test suite. # https://github.com/vshabanov/HsOpenSSL/issues/44 @@ -910,7 +1085,7 @@ let # servant-server # Some getting: *** abort because of serious configure-time warning from Cabal (multiple different package versions in project) # stack2nix - Cabal = + Cabal = if !areCabalPatchesRequired then super.Cabal else # no patches needed -> don't even try to access any attributes if approach == "pkgsMusl" then ( # Example package where this matters: `focuslist` # See note [When Cabal is `null` in a package set]. @@ -945,6 +1120,9 @@ let # It's not clear if it's safe to disable this as key functionality may be broken hslua = dontCheck super.hslua; + # Test suite takes > 1h CPU time with 1600% CPU on my CI machine. + hw-balancedparens = dontCheck super.hw-balancedparens; + # Test suite tries to connect to dbus, can't work in sandbox. credential-store = dontCheck super.credential-store; @@ -989,6 +1167,13 @@ let pcre-light = super.pcre-light.override { pcre = final.pcre_static; }; bzlib-conduit = super.bzlib-conduit.override { bzip2 = final.bzip2_static; }; + # Tests fail with: doctests: : Dynamic loading not supported + BNFC = dontCheck super.BNFC; + + # 200 ms test timeout is not suitable for massively parallel CI. + # See https://github.com/jensblanck/cdar/issues/7 + cdar-mBound = dontCheck super.cdar-mBound; + darcs = addStaticLinkerFlagsWithPkgconfig # (super.darcs.override { curl = curl_static; }) @@ -1006,34 +1191,51 @@ let # Currently fails with linker error, see `yesod-paginator` below. erd = doJailbreak super.erd; - hmatrix = ((drv: enableCabalFlag drv "no-random_r") (overrideCabal super.hmatrix (old: { - # The patch does not apply cleanly because the cabal file - # was Hackage-revisioned, which converted it to Windows line endings - # (https://github.com/haskell-numerics/hmatrix/issues/302); - # convert it back. - prePatch = (old.prePatch or "") + '' - ${final.dos2unix}/bin/dos2unix ${old.pname}.cabal - ''; - patches = (old.patches or []) ++ [ - (final.fetchpatch { - url = "https://github.com/nh2/hmatrix/commit/e9da224bce287653f96235bd6ae02da6f8f8b219.patch"; - name = "hmatrix-Allow-disabling-random_r-usage-manually.patch"; - sha256 = "1fpv0y5nnsqcn3qi767al694y01km8lxiasgwgggzc7816xix0i2"; - stripLen = 2; - }) - ]; - }))).override { openblasCompat = final.openblasCompat; }; + # Test timeout is too tight on `-O0`. + Glob = + (if disableOptimization then dontCheck else lib.id) + super.Glob; + + # Tests fail with: doctests: : Dynamic loading not supported + headroom = dontCheck super.headroom; + + hmatrix = + # musl does not have `random_r()`. + (enableCabalFlag super.hmatrix "no-random_r") + .override { openblasCompat = final.openblasCompat; }; + + # Tests fail with: doctests: : Dynamic loading not supported + hw-xml = dontCheck super.hw-xml; + hw-packed-vector = dontCheck super.hw-packed-vector; # Test suite segfaults (perhaps because R's test suite also does?). inline-r = dontCheck super.inline-r; + # Tests fail with: doctests: : Dynamic loading not supported + openapi3 = dontCheck super.openapi3; + # TODO For the below packages, it would be better if we could somehow make all users # of postgresql-libpq link in openssl via pkgconfig. + hasql-notifications = + addStaticLinkerFlagsWithPkgconfig + super.hasql-notifications + [ final.openssl final.postgresql ] + "--libs libpq"; + hasql-queue = + addStaticLinkerFlagsWithPkgconfig + super.hasql-queue + [ final.openssl final.postgresql ] + "--libs libpq"; pg-harness-server = addStaticLinkerFlagsWithPkgconfig super.pg-harness-server [ final.openssl final.postgresql ] "--libs libpq"; + postgrest = + addStaticLinkerFlagsWithPkgconfig + super.postgrest + [ final.openssl final.postgresql ] + "--libs libpq"; postgresql-orm = addStaticLinkerFlagsWithPkgconfig super.postgresql-orm @@ -1054,6 +1256,11 @@ let super.squeal-postgresql [ final.openssl ] "--libs openssl"; + tmp-postgres = + addStaticLinkerFlagsWithPkgconfig + (dontCheck super.tmp-postgres) # flaky/stuck tests: https://github.com/jfischoff/tmp-postgres/issues/273 + [ final.openssl ] + "--libs openssl"; xml-to-json = addStaticLinkerFlagsWithPkgconfig @@ -1136,6 +1343,26 @@ let ]) "--libs nettle sdl2 SDL2_image xcursor libpng libjpeg libtiff-4 libwebp"; + # With optimisations disabled, some tests of its test suite don't + # finish within the 25 seconds timeout. + skylighting-core = + (if disableOptimization then dontCheck else lib.id) + super.skylighting-core; + + # Test suite loops forever without optimisations.. + text-short = + (if disableOptimization then dontCheck else lib.id) + super.text-short; + + # Flaky QuickCheck test failure: + # *** Failed! "00:01": expected Just 00:00:60, found Just 00:01:00 (after 95 tests and 2 shrinks): + # See https://github.com/haskellari/time-compat/issues/23 + time-compat = dontCheck super.time-compat; + + # Test suite takes > 1h CPU time with 1600% CPU on my CI machine. + # Does pass after that time though, maybe high thread counds work badly here. + tomland = dontCheck super.tomland; + # Added for #14 tttool = callCabal2nix "tttool" (final.fetchFromGitHub { owner = "entropia"; @@ -1225,6 +1452,53 @@ let dontCheck (overrideCabal super.hakyll (drv: { testToolDepends = []; })); + + # Inspection tests fail on `disableOptimization`with + # examples/Fusion.hs:25:1: sumUpSort `hasNoType` GHC.Types.[] failed expectedly. + inspection-testing = + (if disableOptimization then dontCheck else lib.id) + super.inspection-testing; + + # Inspection tests fail on `disableOptimization`with + # examples/Fusion.hs:25:1: sumUpSort `hasNoType` GHC.Types.[] failed expectedly + algebraic-graphs = + (if disableOptimization then dontCheck else lib.id) + super.algebraic-graphs; + + # Test suite tries to connect to the Internet + aur = dontCheck super.aur; + + # Test suite tries to run `minisat` which is not on PATH + ersatz = dontCheck super.ersatz; + + # Seems to time out on `-O0` (but does not print that timeout + # is the failure reason). + numeric-prelude = + (if disableOptimization then dontCheck else lib.id) + super.numeric-prelude; + + # doctests test suite fails with: + # /build/trifecta-2.1/src/Text/Trifecta/Util/It.hs:61: failure in expression `let keepIt a = Pure a' + # expected: + # but got: /nix/store/xz6sgnl68v00yhfk25cfankpdf7g57cs-binutils-2.31.1/bin/ld: warning: type and size of dynamic symbol `TextziTrifectaziDelta_zdfHasDeltaByteString_closure' are not defined + trifecta = dontCheck super.trifecta; + + # Test suite needs a missing dependency: + # Configuring range-set-list-0.1.3.1... + # ... + # Setup: Encountered missing or private dependencies: + # tasty >=0.8 && <1.4 + range-set-list = + dontCheck (overrideCabal super.range-set-list { broken = false; }); + + # Test suite fails: + # + # doctests: FAIL + # Exception: : Dynamic loading not supported + # Parse a content-less file: FAIL + # Exception: test-files/trivial.proto: openFile: does not exist (No such file or directory) + # 2 out of 71 tests failed (1.97s) + proto3-suite = dontCheck super.proto3-suite; }); }); @@ -1246,19 +1520,29 @@ let # for compiling its `Setup.hs` the Cabal package that comes with GHC # (that is in the default GHC package DB) is used instead, which # obviously doesn' thave our patches. - statify = drv: with final.haskell.lib; final.lib.foldl appendConfigureFlag (disableLibraryProfiling (disableSharedExecutables (useFixedCabal drv))) ([ - "--enable-executable-static" # requires `useFixedCabal` - "--extra-lib-dirs=${final.ncurses.override { enableStatic = true; }}/lib" - # TODO Figure out why this and the below libffi are necessary. - # `working` and `workingStackageExecutables` don't seem to need that, - # but `static-stack2nix-builder-example` does. - ] ++ final.lib.optionals (!integer-simple) [ - "--extra-lib-dirs=${final.gmp6.override { withStatic = true; }}/lib" - ] ++ final.lib.optionals (!integer-simple && approach == "pkgsMusl") [ - # GHC needs this if it itself wasn't already built against static libffi - # (which is the case in `pkgsStatic` only): - "--extra-lib-dirs=${final.libffi}/lib" - ]); + statify = drv: + with final.haskell.lib; + final.lib.foldl + appendConfigureFlag + (disableLibraryProfiling (disableSharedExecutables (useFixedCabal drv))) + (builtins.concatLists [ + [ + "--enable-executable-static" # requires `useFixedCabal` + # `enableShared` seems to be required to avoid `recompile with -fPIC` errors on some packages. + "--extra-lib-dirs=${final.ncurses.override { enableStatic = true; enableShared = true; }}/lib" + ] + # TODO Figure out why this and the below libffi are necessary. + # `working` and `workingStackageExecutables` don't seem to need that, + # but `static-stack2nix-builder-example` does. + (final.lib.optionals (!integer-simple) [ + "--extra-lib-dirs=${final.gmp6.override { withStatic = true; }}/lib" + ]) + (final.lib.optionals (!integer-simple && approach == "pkgsMusl") [ + # GHC needs this if it itself wasn't already built against static libffi + # (which is the case in `pkgsStatic` only): + "--extra-lib-dirs=${final.libffi}/lib" + ]) + ]); in final.lib.mapAttrs (name: value: @@ -1290,7 +1574,11 @@ in cabal-install bench dhall + dhall-json + postgrest + proto3-suite hsyslog # Small example of handling https://github.com/NixOS/nixpkgs/issues/43849 correctly + # aura # `aur` maked as broken in nixpkgs, but works here with `allowBroken = true;` actually ; } // (if approach == "pkgsStatic" then {} else { # Packages that work with `pkgsMusl` but fail with `pkgsStatic`: @@ -1316,7 +1604,6 @@ in notWorking = { inherit (haskellPackages) - aura # Removed for now as it keeps having Cabal bounds issues (https://github.com/aurapm/aura/issues/526#issuecomment-493716675) tttool # see #14 # TODO reenable after fixing Package `HPDF-1.4.10` being marked as broken and failing to evaluate ; }; @@ -1332,26 +1619,25 @@ in builtins.removeAttrs allStackageExecutables [ # List of executables that don't work for reasons not yet investigated. # When changing this file, we should always check if this list grows or shrinks. - "Agda" # anonymous function at build-support/fetchurl/boot.nix:5:1 called with unexpected argument 'meta', at build-support/fetchpatch/default.nix:14:1 - "Allure" # marked as broken - "csg" # marked as broken + "Agda" # fails on `emacs` not building + "Allure" # depends on `LambdaHack` also in this list + "csg" # `base >=4.0 && <4.14` on `doctest-driver-gen` "cuda" # needs `allowUnfree = true`; enabling it gives `unsupported platform for the pure Linux stdenv` - "debug" # marked as broken - "diagrams-builder" # marked as broken - "ersatz" # marked as broken - "gloss-examples" # needs opengl: `cannot find -lGLU` `-lGL` - "gtk3" # problem compiling `glib` dependency: relocation R_X86_64_32 against hidden symbol `__TMC_END__' can not be used when making a PIE object - "H" # `zgemm_: symbol not found` when compiling Main; not clear how that can be provided - "hamilton" # openmp linker error via openblas - "hquantlib" # marked as broken - "ihaskell" # marked as broken - "LambdaHack" # marked as broken - "language-puppet" # dependency `hruby` does not build + "debug" # `regex-base <0.94` on `regex-tdfa-text` + "diagrams-builder" # `template-haskell >=2.5 && <2.16` on `size-based` + "gloss-examples" # `base >=4.8 && <4.14` on `repa-io` + "gtk3" # Haskell package `glib` fails with `Ambiguous module name ‘Gtk2HsSetup’: it was found in multiple packages: gtk2hs-buildtools-0.13.8.0 gtk2hs-buildtools-0.13.8.0` + "H" # error: anonymous function at pkgs/applications/science/math/R/default.nix:1:1 called with unexpected argument 'javaSupport', at lib/customisation.nix:69:16 + "hamilton" # `_gfortran_concat_string` linker error via openblas + "hquantlib" # `time >=1.4.0.0 && <1.9.0.0` on `hquantlib-time` + "ihaskell" # linker error + "LambdaHack" # fails `systemd` dependency erroring on `#include ` + "language-puppet" # `base >=4.6 && <4.14, ghc-prim >=0.3 && <0.6` for dependency `protolude` "learn-physics" # needs opengl: `cannot find -lGLU` `-lGL` - "odbc" # marked as broken - "qchas" # openmp linker error via openblas + "odbc" # `odbcss.h: No such file or directory` + "qchas" # `_gfortran_concat_string` linker error via openblas "rhine-gloss" # needs opengl: `cannot find -lGLU` `-lGL` - "soxlib" # dependency `sox` fails with: `formats.c:425:4: error: #error FIX NEEDED HERE` + "soxlib" # fails `systemd` dependency erroring on `#include ` ]; inherit normalPkgs;