Merge pull request IntersectMBO#5645 from IntersectMBO/smelc/testnet-allow-passing-config-files

cardano-testnet: allow to pass genesis files

Author: smelc

cardano-node-chairman/test/Spec/Chairman/Cardano.hs

@@ -16,6 +16,6 @@ hprop_chairman :: H.Property
 hprop_chairman = H.integrationRetryWorkspace 2 "cardano-chairman" $ \tempAbsPath' -> do
   conf <- H.mkConf tempAbsPath'
 
-  allNodes <- fmap H.nodeName . H.allNodes <$> H.cardanoTestnet H.cardanoDefaultTestnetOptions conf
+  allNodes <- fmap H.nodeName . H.allNodes <$> H.cardanoTestnetDefault H.cardanoDefaultTestnetOptions conf
 
   chairmanOver 120 50 conf allNodes

cardano-testnet/src/Cardano/Testnet.hs

@@ -5,6 +5,7 @@ module Cardano.Testnet (
 
   -- ** Start a testnet
   cardanoTestnet,
+  cardanoTestnetDefault,
 
   -- ** Testnet options
   CardanoTestnetOptions(..),

cardano-testnet/src/Parsers/Run.hs

@@ -52,4 +52,4 @@ runTestnetCmd = \case
 
 runCardanoOptions :: CardanoTestnetOptions -> IO ()
 runCardanoOptions options =
-  runTestnet $ cardanoTestnet options
+  runTestnet $ cardanoTestnetDefault options

cardano-testnet/src/Testnet/Components/Configuration.hs

@@ -1,4 +1,5 @@
 {-# LANGUAGE GADTs #-}
+{-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE TypeApplications #-}
 
@@ -8,6 +9,8 @@ module Testnet.Components.Configuration
   , createSPOGenesisAndFiles
   , mkTopologyConfig
   , numSeededUTxOKeys
+  , NumPools
+  , numPools
   ) where
 
 import           Cardano.Api.Pretty
@@ -30,7 +33,6 @@ import qualified Data.ByteString.Lazy as LBS
 import           Data.Char (toLower)
 import qualified Data.List as List
 import           Data.String
-import           Data.Time
 import           GHC.Stack (HasCallStack)
 import qualified GHC.Stack as GHC
 import           Lens.Micro
@@ -42,11 +44,13 @@ import qualified Hedgehog.Extras.Stock.Time as DTC
 import qualified Hedgehog.Extras.Test.Base as H
 import qualified Hedgehog.Extras.Test.File as H
 
+import           Cardano.Api.Ledger (StandardCrypto)
+import           Data.Word (Word32)
 import           Testnet.Defaults
 import           Testnet.Filepath
 import           Testnet.Process.Run (execCli_)
 import           Testnet.Property.Utils
-import           Testnet.Start.Types
+import           Testnet.Start.Types (CardanoTestnetOptions (..))
 
 
 createConfigYaml
@@ -78,35 +82,35 @@ createConfigYaml (TmpAbsolutePath tempAbsPath') anyCardanoEra' = GHC.withFrozenC
 numSeededUTxOKeys :: Int
 numSeededUTxOKeys = 3
 
+newtype NumPools = NumPools Int
+
+numPools :: CardanoTestnetOptions -> NumPools
+numPools CardanoTestnetOptions { cardanoNodes } = NumPools $ length cardanoNodes
+
 createSPOGenesisAndFiles
   :: (MonadTest m, MonadCatch m, MonadIO m, HasCallStack)
-  => CardanoTestnetOptions
-  -> UTCTime -- ^ Start time
+  => NumPools -- ^ The number of pools to make
+  -> AnyCardanoEra -- ^ The era to use
+  -> ShelleyGenesis StandardCrypto -- ^ The shelley genesis to use.
   -> TmpAbsolutePath
   -> m FilePath -- ^ Shelley genesis directory
-createSPOGenesisAndFiles testnetOptions startTime (TmpAbsolutePath tempAbsPath') = do
+createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis (TmpAbsolutePath tempAbsPath') = do
   let genesisShelleyFpAbs = tempAbsPath' </> defaultShelleyGenesisFp
       genesisShelleyDirAbs = takeDirectory genesisShelleyFpAbs
   genesisShelleyDir <- H.createDirectoryIfMissing genesisShelleyDirAbs
-  let testnetMagic = cardanoTestnetMagic testnetOptions
-      numPoolNodes = length $ cardanoNodes testnetOptions
+  let testnetMagic = sgNetworkMagic shelleyGenesis
       numStakeDelegators = 3
-      era = cardanoNodeEra testnetOptions
-      -- TODO: Even this is cumbersome. You need to know where to put the initial
-      -- shelley genesis for create-testnet-data to use.
+      startTime = sgSystemStart shelleyGenesis
 
   -- TODO: We need to read the genesis files into Haskell and modify them
   -- based on cardano-testnet's cli parameters
 
   -- We create the initial genesis file to avoid having to re-write the genesis file later
   -- with the parameters we want. The user must provide genesis files or we will use a default.
-  -- We should *never* be modifying the genesis file after cardano-testnet is run because this
+  -- We should *never* be modifying the genesis file after @cardanoTestnet@ is run because this
   -- is sure to be a source of confusion if users provide genesis files and we are mutating them
   -- without their knowledge.
-  let shelleyGenesis :: LBS.ByteString
-      shelleyGenesis = encode $ defaultShelleyGenesis startTime testnetOptions
-
-  H.evalIO $ LBS.writeFile genesisShelleyFpAbs shelleyGenesis
+  H.evalIO $ LBS.writeFile genesisShelleyFpAbs $ encode shelleyGenesis
 
   -- TODO: Remove this rewrite.
  -- 50 second epochs
@@ -128,7 +132,7 @@ createSPOGenesisAndFiles testnetOptions startTime (TmpAbsolutePath tempAbsPath')
   execCli_
     [ convertToEraString era, "genesis", "create-testnet-data"
     , "--spec-shelley", genesisShelleyFpAbs
-    , "--testnet-magic", show @Int testnetMagic
+    , "--testnet-magic", show @Word32 testnetMagic
     , "--pools", show @Int numPoolNodes
     , "--supply", "1000000000000"
     , "--supply-delegated", "1000000000000"
@@ -138,7 +142,7 @@ createSPOGenesisAndFiles testnetOptions startTime (TmpAbsolutePath tempAbsPath')
     , "--out-dir", tempAbsPath'
     ]
 
-  -- Here we move all of the keys etc generated by create-staked
+  -- Here we move all of the keys etc generated by create-testnet-data
   -- for the nodes to use
 
   -- Move all genesis related files

cardano-testnet/src/Testnet/Start/Cardano.hs

@@ -16,13 +16,11 @@ module Testnet.Start.Cardano
   , PaymentKeyPair(..)
 
   , cardanoTestnet
-
+  , cardanoTestnetDefault
   ) where
 
 
 import           Control.Monad
-import qualified Control.Monad.Class.MonadTimer.SI as MT
-import           Control.Monad.IO.Class
 import           Control.Monad.Trans.Class (lift)
 import           Control.Monad.Trans.Except (runExceptT)
 import           Data.Aeson
@@ -43,14 +41,25 @@ import qualified Hedgehog.Extras.Stock.OS as OS
 import qualified Hedgehog.Extras.Test.Base as H
 import qualified Hedgehog.Extras.Test.File as H
 
+import qualified Testnet.Defaults as Defaults
+
+import           Cardano.Api
+import           Cardano.Api.Ledger (StandardCrypto)
+import           Cardano.Ledger.Alonzo.Genesis (AlonzoGenesis)
+import           Cardano.Ledger.Conway.Genesis (ConwayGenesis)
+import qualified Control.Monad.Class.MonadTimer.SI as MT
+import           Control.Monad.IO.Class
+import qualified Data.Aeson as Aeson
+import           Data.Bifunctor (first)
+import           Data.Time (UTCTime)
+import           Data.Word (Word32)
 import           Testnet.Components.Configuration
-import           Testnet.Defaults
 import           Testnet.Filepath
 import qualified Testnet.Process.Run as H
 import           Testnet.Process.Run
 import qualified Testnet.Property.Assert as H
 import           Testnet.Property.Checks
-import           Testnet.Runtime
+import           Testnet.Runtime as TR hiding (shelleyGenesis)
 import qualified Testnet.Start.Byron as Byron
 import           Testnet.Start.Types
 
@@ -63,10 +72,11 @@ import           Testnet.Start.Types
 -- a valid node cluster.
 testnetMinimumConfigurationRequirements :: CardanoTestnetOptions -> H.Integration ()
 testnetMinimumConfigurationRequirements cTestnetOpts = do
-  when (length (cardanoNodes cTestnetOpts) < 2) $ do
-     H.noteShow_ ("Need at least two nodes to run a cluster" :: String)
+  let actualLength = length (cardanoNodes cTestnetOpts)
+  when (actualLength < 2) $ do
+     H.noteShow_ ("Need at least two nodes to run a cluster, but got: " <> show actualLength)
      H.noteShow_ cTestnetOpts
-     H.assert False
+     H.failure
 
 data ForkPoint
   = AtVersion Int
@@ -79,6 +89,19 @@ data ForkPoint
 startTimeOffsetSeconds :: DTC.NominalDiffTime
 startTimeOffsetSeconds = if OS.isWin32 then 90 else 15
 
+-- | Like 'cardanoTestnet', but using defaults for all configuration files.
+-- See 'cardanoTestnet' for additional documentation.
+cardanoTestnetDefault :: ()
+  => CardanoTestnetOptions
+  -> Conf
+  -> H.Integration TestnetRuntime
+cardanoTestnetDefault opts conf = do
+  alonzoGenesis <- H.evalEither $ first prettyError Defaults.defaultAlonzoGenesis
+  currentTime <- H.noteShowIO DTC.getCurrentTime
+  startTime <- H.noteShow $ DTC.addUTCTime startTimeOffsetSeconds currentTime
+  cardanoTestnet
+    opts conf startTime
+    (Defaults.defaultShelleyGenesis startTime opts) alonzoGenesis Defaults.defaultConwayGenesis
 
 -- | Setup a number of credentials and pools, like this:
 --
@@ -122,13 +145,37 @@ startTimeOffsetSeconds = if OS.isWin32 then 90 else 15
 -- > │   └── node-spo{1,2,3}
 -- > └── utxo-keys
 -- >     └── utxo{1,2,3}.{addr,skey,vkey}
-cardanoTestnet :: CardanoTestnetOptions -> Conf -> H.Integration TestnetRuntime
-cardanoTestnet testnetOptions Conf {tempAbsPath=TmpAbsolutePath tmpAbsPath} = do
-  testnetMinimumConfigurationRequirements testnetOptions
-  void $ H.note OS.os
-  currentTime <- H.noteShowIO DTC.getCurrentTime
-  let testnetMagic = cardanoTestnetMagic testnetOptions
+cardanoTestnet :: ()
+  => CardanoTestnetOptions -- ^ The options to use. Must be consistent with the genesis files.
+  -> Conf
+  -> UTCTime -- ^ The starting time. Must be the same as the one in the shelley genesis.
+  -> ShelleyGenesis StandardCrypto -- ^ The shelley genesis to use, for example 'Defaults.defaultShelleyGenesis'.
+                                   --   Some fields are overridden by the accompanying 'CardanoTestnetOptions'.
+  -> AlonzoGenesis -- ^ The alonzo genesis to use, for example 'Defaults.defaultAlonzoGenesis'.
+  -> ConwayGenesis StandardCrypto -- ^ The conway genesis to use, for example 'Defaults.defaultConwayGenesis'.
+  -> H.Integration TestnetRuntime
+cardanoTestnet
+  testnetOptions Conf {tempAbsPath=TmpAbsolutePath tmpAbsPath} startTime
+  shelleyGenesis alonzoGenesis conwayGenesis = do
+  let shelleyStartTime = sgSystemStart shelleyGenesis
+      shelleyTestnetMagic = sgNetworkMagic shelleyGenesis
+      optionsMagic :: Word32 = fromIntegral $ cardanoTestnetMagic testnetOptions
+      testnetMagic = cardanoTestnetMagic testnetOptions
       numPoolNodes = length $ cardanoNodes testnetOptions
+      nbPools = numPools testnetOptions
+      era = cardanoNodeEra testnetOptions
+
+   -- Sanity checks
+  testnetMinimumConfigurationRequirements testnetOptions
+  when (shelleyStartTime /= startTime) $ do
+    H.note_ $ "Expected same system start in shelley genesis and parameter, but got " <> show shelleyStartTime <> " and " <> show startTime
+    H.failure
+  when (shelleyTestnetMagic /= optionsMagic) $ do
+    H.note_ $ "Expected same network magic in shelley genesis and parameter, but got " <> show shelleyTestnetMagic <> " and " <> show optionsMagic
+    H.failure
+  -- Done with sanity checks
+
+  H.note_ OS.os
 
   if all isJust [mconfig | SpoTestnetNodeOptions mconfig _ <- cardanoNodes testnetOptions]
   then
@@ -138,10 +185,8 @@ cardanoTestnet testnetOptions Conf {tempAbsPath=TmpAbsolutePath tmpAbsPath} = do
     -- See all of the ad hoc file creation/renaming/dir creation etc below.
     H.failMessage GHC.callStack "Specifying node configuration files per node not supported yet."
   else do
-    startTime <- H.noteShow $ DTC.addUTCTime startTimeOffsetSeconds currentTime
-
     H.lbsWriteFile (tmpAbsPath </> "byron.genesis.spec.json")
-      . encode $ defaultByronProtocolParamsJsonValue
+      . encode $ Defaults.defaultByronProtocolParamsJsonValue
 
     -- Because in Conway the overlay schedule and decentralization parameter
     -- are deprecated, we must use the "create-staked" cli command to create
@@ -153,10 +198,18 @@ cardanoTestnet testnetOptions Conf {tempAbsPath=TmpAbsolutePath tmpAbsPath} = do
       (tmpAbsPath </> "byron.genesis.spec.json")
       (tmpAbsPath </> "byron-gen-command")
 
-    _ <- createSPOGenesisAndFiles testnetOptions startTime (TmpAbsolutePath tmpAbsPath)
+    -- Write Alonzo genesis file
+    alonzoGenesisJsonFile <- H.noteShow $ tmpAbsPath </> "genesis.alonzo.spec.json"
+    H.evalIO $ LBS.writeFile alonzoGenesisJsonFile $ Aeson.encode alonzoGenesis
+
+    -- Write Conway genesis file
+    conwayGenesisJsonFile <- H.noteShow $ tmpAbsPath </> "genesis.conway.spec.json"
+    H.evalIO $ LBS.writeFile conwayGenesisJsonFile $ Aeson.encode conwayGenesis
 
     configurationFile <- H.noteShow $ tmpAbsPath </> "configuration.yaml"
 
+    _ <- createSPOGenesisAndFiles nbPools era shelleyGenesis (TmpAbsolutePath tmpAbsPath)
+
     poolKeys <- H.noteShow $ flip fmap [1..numPoolNodes] $ \n ->
       PoolNodeKeys
         { poolNodeKeysColdVkey = tmpAbsPath </> "pools" </> "cold" <> show n <> ".vkey"
@@ -213,7 +266,7 @@ cardanoTestnet testnetOptions Conf {tempAbsPath=TmpAbsolutePath tmpAbsPath} = do
 
 
     -- Add Byron, Shelley and Alonzo genesis hashes to node configuration
-    finalYamlConfig <- createConfigYaml (TmpAbsolutePath tmpAbsPath) $ cardanoNodeEra testnetOptions
+    finalYamlConfig <- createConfigYaml (TmpAbsolutePath tmpAbsPath) era
 
     H.evalIO $ LBS.writeFile configurationFile finalYamlConfig
 
@@ -337,7 +390,7 @@ cardanoTestnet testnetOptions Conf {tempAbsPath=TmpAbsolutePath tmpAbsPath} = do
 
       let runtime = TestnetRuntime
             { configurationFile
-            , shelleyGenesisFile = tmpAbsPath </> defaultShelleyGenesisFp
+            , shelleyGenesisFile = tmpAbsPath </> Defaults.defaultShelleyGenesisFp
             , testnetMagic
             , poolNodes
             , wallets = wallets

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Babbage/LeadershipSchedule.hs

@@ -77,7 +77,7 @@ hprop_leadershipSchedule = H.integrationRetryWorkspace 2 "babbage-leadership-sch
     { testnetMagic
     , wallets
     , configurationFile
-    } <- cardanoTestnet cTestnetOptions conf
+    } <- cardanoTestnetDefault cTestnetOptions conf
 
   node1sprocket <- H.headM $ poolSprockets tr
   execConfig <- H.mkExecConfig tempBaseAbsPath node1sprocket testnetMagic

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Babbage/StakeSnapshot.hs

@@ -62,7 +62,7 @@ hprop_stakeSnapshot = H.integrationRetryWorkspace 2 "babbage-stake-snapshot" $ \
   TestnetRuntime
     { testnetMagic
     , poolNodes
-    } <- cardanoTestnet options conf
+    } <- cardanoTestnetDefault options conf
 
   poolNode1 <- H.headM poolNodes
   poolSprocket1 <- H.noteShow $ nodeSprocket $ poolRuntime poolNode1

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Babbage/Transaction.hs

@@ -66,7 +66,7 @@ hprop_transaction = H.integrationRetryWorkspace 0 "babbage-transaction" $ \tempA
     { testnetMagic
     , poolNodes
     , wallets
-    } <- cardanoTestnet options conf
+    } <- cardanoTestnetDefault options conf
 
   poolNode1 <- H.headM poolNodes
   poolSprocket1 <- H.noteShow $ nodeSprocket $ poolRuntime poolNode1

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Conway/StakeSnapshot.hs

@@ -62,7 +62,7 @@ hprop_stakeSnapshot = H.integrationRetryWorkspace 2 "conway-stake-snapshot" $ \t
   TestnetRuntime
     { testnetMagic
     , poolNodes
-    } <- cardanoTestnet options conf
+    } <- cardanoTestnetDefault options conf
 
   poolNode1 <- H.headM poolNodes
   poolSprocket1 <- H.noteShow $ nodeSprocket $ poolRuntime poolNode1

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/KesPeriodInfo.hs

@@ -65,7 +65,7 @@ hprop_kes_period_info = H.integrationRetryWorkspace 2 "kes-period-info" $ \tempA
                           , cardanoNodeEra = AnyCardanoEra era -- TODO: We should only support the latest era and the upcoming era
                           }
 
-  runTime@TestnetRuntime { configurationFile, testnetMagic, wallets } <- cardanoTestnet cTestnetOptions conf
+  runTime@TestnetRuntime { configurationFile, testnetMagic, wallets } <- cardanoTestnetDefault cTestnetOptions conf
   node1sprocket <- H.headM $ poolSprockets runTime
   execConfig <- H.mkExecConfig tempBaseAbsPath node1sprocket testnetMagic