Merge #1587 #1593

1587: Cabal: Set -Werror and -Ox only with release flag r=rvl a=rvl

Makes Cabal builds faster, like the `stack build --fast` flag.

Nothing is changed when the development flag is unset.

Some components are missing `-O`, and most others are at `-O2`. However, from what I've read, `-O2` often doesn't help, so you only use it if your benchmarks say so.

1593: cardano-node: 1.10.1 -> 1.11.0 r=rvl a=rvl

# Issue Number

Relates to #1577 / ADP-83.

# Overview

- Updates cardano-node version in nix.
- Updates the cardano-haskell stack snapshot.
- Adds `cardano-cli` to the shell. I am hoping to use this in the integration tests.
- Build fixes

# Comments

WIP until input-output-hk/cardano-haskell#13 is merged.


Co-authored-by: Rodney Lorrimar <rodney.lorrimar@iohk.io>
Co-authored-by: IOHK <devops+stack-project@iohk.io>

Author: 3 people

.buildkite/pipeline.yml

@@ -21,7 +21,10 @@ steps:
       system: x86_64-linux
 
   - label: 'Check Cabal Configure'
-    command: 'nix-shell --run "cabal v2-update && cabal v2-configure --enable-tests --enable-benchmarks"'
+    command:
+      - 'nix-shell --run "cabal v2-update"'
+      - 'nix-shell --run "cabal v2-configure --enable-tests --enable-benchmarks"'
+      - 'nix-shell --run "cabal v2-configure -frelease --enable-tests --enable-benchmarks"'
     agents:
       system: x86_64-linux
 

.buildkite/rebuild.hs

@@ -137,16 +137,19 @@ parseOpts = execParser opts
         )
 
 buildStep :: DryRun -> Maybe BuildkiteEnv -> Bool -> IO ExitCode
-buildStep dryRun bk nightly =
+buildStep dryRun bk nightly = do
+    pkgs <- listLocalPackages
+    let cabalFlags = concatMap (flag "release") pkgs
+
     titled "Build LTS Snapshot"
         (build Standard ["--only-snapshot"]) .&&.
-    titled "Build dependencies"
+      titled "Build dependencies"
         (build Standard ["--only-dependencies"]) .&&.
-    titled "Build"
-        (build Fast ["--test", "--no-run-tests"]) .&&.
-    titled "Test"
-        (timeout 45 (test Fast Serial .&&. test Fast Parallel)) .&&.
-    titled "Checking golden test files"
+      titled "Build"
+        (build Fast (["--test", "--no-run-tests"] ++ cabalFlags)) .&&.
+      titled "Test"
+        (timeout 45 (test Fast Serial cabalFlags .&&. test Fast Parallel cabalFlags)) .&&.
+      titled "Checking golden test files"
         (checkUnclean dryRun "lib/core/test/data")
   where
     build opt args =
@@ -162,7 +165,7 @@ buildStep dryRun bk nightly =
             , args
             ]
 
-    test opt behavior =
+    test opt behavior args =
         run dryRun "stack" $ concat
             [ color "always"
             , [ "test" ]
@@ -176,6 +179,7 @@ buildStep dryRun bk nightly =
                     ta (match serialTests ++ jobs 1) ++ jobs 1
                 Parallel ->
                     ta (skip serialTests)
+            , args
             ]
 
     color arg = ["--color", arg]
@@ -184,6 +188,7 @@ buildStep dryRun bk nightly =
     skip  arg = ["--skip", arg]
     match arg = ["--match", arg]
     ta    arg = ["--ta", T.unwords arg]
+    flag name pkg = ["--flag", pkg <> ":" <> name]
 
     serialTests = "SERIAL"
 
@@ -216,6 +221,17 @@ checkUnclean dryRun dir = do
         printf "This build step will fail.\n"
     pure res
 
+-- | Find the name of all Cabal packages in this repo.
+listLocalPackages :: IO [Text]
+listLocalPackages =
+    find (suffix ".cabal") "."
+    & onFiles (grepText (invert isBuild))
+    & fmap basename
+    & textLines
+  where
+    isBuild = contains "./.stack-work" <|> contains "./dist"
+    textLines shell = fold (format fp <$> shell) Fold.list
+
 ----------------------------------------------------------------------------
 -- Buildkite
 -- https://buildkite.com/docs/pipelines/environment-variables

default.nix

@@ -77,7 +77,7 @@ let
     inherit pkgs commonLib src haskellPackages stackNixRegenerate;
     inherit (jmPkgs) jormungandr jormungandr-cli;
     # expose cardano-node, so daedalus can ship it without needing to pin cardano-node
-    inherit (pkgs) cardano-node;
+    inherit (pkgs) cardano-node cardano-cli;
     inherit (haskellPackages.cardano-wallet-core.identifier) version;
     # expose db-converter, so daedalus can ship it without needing to pin a ouroborus-network rev
     inherit (haskellPackages.ouroboros-consensus-byron.components.exes) db-converter;
@@ -133,6 +133,7 @@ let
           jormungandr
           jormungandr-cli
           cardano-node
+          cardano-cli
         ]) ++ (with pkgs; [
           stack
           cabal-install

lib/byron/cardano-wallet-byron.cabal

@@ -11,8 +11,8 @@ category:            Web
 build-type:          Simple
 cabal-version:       >=1.10
 
-flag development
-    description: Disable `-Werror`
+flag release
+    description: Enable optimization and `-Werror`
     default: False
     manual: True
 
@@ -26,9 +26,8 @@ library
       -Wall
       -Wcompat
       -fwarn-redundant-constraints
-  if (!flag(development))
-    ghc-options:
-      -Werror
+  if (flag(release))
+    ghc-options: -O2 -Werror
   build-depends:
       base
     , aeson
@@ -91,10 +90,8 @@ executable cardano-wallet-byron
   ghc-options:
       -threaded -rtsopts
       -Wall
-      -O2
-  if (!flag(development))
-    ghc-options:
-      -Werror
+  if (flag(release))
+    ghc-options: -O2 -Werror
   build-depends:
       base
     , cardano-wallet-byron
@@ -122,10 +119,8 @@ test-suite unit
   ghc-options:
       -threaded -rtsopts
       -Wall
-  if (!flag(development))
-    ghc-options:
-      -O2
-      -Werror
+  if (flag(release))
+    ghc-options: -O2 -Werror
   build-depends:
       base
     , bytestring
@@ -164,10 +159,8 @@ test-suite integration
   ghc-options:
       -threaded -rtsopts
       -Wall
-  if (!flag(development))
-    ghc-options:
-      -O2
-      -Werror
+  if (flag(release))
+    ghc-options: -O2 -Werror
   build-depends:
       base
     , aeson
@@ -204,94 +197,83 @@ test-suite integration
       Test.Integration.Byron.Scenario.CLI.Transactions
 
 benchmark restore
-   default-language:
-       Haskell2010
-   default-extensions:
-       NoImplicitPrelude
-       OverloadedStrings
-   ghc-options:
-       -threaded -rtsopts
-       -Wall
-       -O2
-   if (!flag(development))
-     ghc-options:
-       -Werror
-   build-depends:
-       base
-     , bytestring
-     , cardano-wallet-core
-     , optparse-applicative
-     , cardano-wallet-byron
-     , cardano-wallet-launcher
-     , ouroboros-network
-     , contra-tracer
-     , containers
-     , criterion-measurement
-     , cryptonite
-     , deepseq
-     , digest
-     , filepath
-     , fmt
-     , iohk-monitoring
-     , persistent
-     , persistent-template
-     , process
-     , say
-     , temporary
-     , text
-     , time
-     , transformers
-   type:
-      exitcode-stdio-1.0
-   hs-source-dirs:
-       bench
-   main-is:
-       Restore.hs
-   other-modules:
-       Cardano.Wallet.Primitive.AddressDiscovery.Any
-       Cardano.Wallet.Primitive.AddressDiscovery.Any.TH
-
-benchmark latency
-   default-language:
-       Haskell2010
-   default-extensions:
-       NoImplicitPrelude
-       OverloadedStrings
-   ghc-options:
-       -threaded -rtsopts
-       -Wall
-       -O2
-   if (!flag(development))
-     ghc-options:
-       -Werror
-   build-depends:
+  default-language:
+      Haskell2010
+  default-extensions:
+      NoImplicitPrelude
+      OverloadedStrings
+  ghc-options:
+      -threaded -rtsopts
+      -Wall
+  if (flag(release))
+    ghc-options: -O2 -Werror
+  build-depends:
       base
-    , aeson
-    , async
-    , bech32
-    , bech32-th
     , bytestring
-    , cardano-wallet-cli
     , cardano-wallet-core
-    , cardano-wallet-core-integration
+    , optparse-applicative
     , cardano-wallet-byron
     , cardano-wallet-launcher
-    , cardano-wallet-test-utils
-    , command
-    , directory
+    , ouroboros-network
+    , contra-tracer
+    , containers
+    , criterion-measurement
+    , cryptonite
+    , deepseq
+    , digest
     , filepath
     , fmt
-    , generic-lens
-    , http-client
-    , http-types
-    , hspec
     , iohk-monitoring
-    , memory
-    , stm
+    , persistent
+    , persistent-template
+    , process
+    , say
     , temporary
     , text
-    , text-class
     , time
+    , transformers
+  type:
+     exitcode-stdio-1.0
+  hs-source-dirs:
+      bench
+  main-is:
+      Restore.hs
+  other-modules:
+      Cardano.Wallet.Primitive.AddressDiscovery.Any
+      Cardano.Wallet.Primitive.AddressDiscovery.Any.TH
+
+benchmark latency
+  default-language:
+      Haskell2010
+  default-extensions:
+      NoImplicitPrelude
+      OverloadedStrings
+  ghc-options:
+      -threaded -rtsopts
+      -Wall
+  if (flag(release))
+    ghc-options: -O2 -Werror
+  build-depends:
+     base
+   , aeson
+   , async
+   , cardano-wallet-cli
+   , cardano-wallet-core
+   , cardano-wallet-core-integration
+   , cardano-wallet-byron
+   , cardano-wallet-launcher
+   , cardano-wallet-test-utils
+   , fmt
+   , generic-lens
+   , http-client
+   , http-types
+   , hspec
+   , iohk-monitoring
+   , stm
+   , temporary
+   , text
+   , text-class
+   , time
   build-tools:
       cardano-wallet-byron
   type:

lib/byron/src/Cardano/Wallet/Byron.hs

@@ -130,10 +130,10 @@ import Network.Wai.Handler.Warp
     ( setBeforeMainLoop )
 import Network.Wai.Middleware.Logging
     ( ApiLog )
+import Ouroboros.Network.CodecCBORTerm
+    ( CodecCBORTerm )
 import Ouroboros.Network.NodeToClient
     ( NodeToClientVersionData (..) )
-import Ouroboros.Network.Protocol.Handshake.Version
-    ( CodecCBORTerm )
 import System.Exit
     ( ExitCode (..) )
 import System.IOManager

lib/byron/src/Cardano/Wallet/Byron/Compatibility.hs

@@ -108,14 +108,14 @@ import Ouroboros.Network.Block
     )
 import Ouroboros.Network.ChainFragment
     ( HasHeader (..) )
+import Ouroboros.Network.CodecCBORTerm
+    ( CodecCBORTerm )
 import Ouroboros.Network.Magic
     ( NetworkMagic (..) )
 import Ouroboros.Network.NodeToClient
     ( NodeToClientVersionData (..), nodeToClientCodecCBORTerm )
 import Ouroboros.Network.Point
     ( WithOrigin (..) )
-import Ouroboros.Network.Protocol.Handshake.Version
-    ( CodecCBORTerm )
 
 import qualified Cardano.Crypto.Hashing as CC
 import qualified Cardano.Wallet.Primitive.Types as W