diff --git a/.gitignore b/.gitignore index 9a09f1b0..0bf7e42d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/.direnv +/.direnv* /.envrc.local result* /.run/ diff --git a/flake.lock b/flake.lock index 5fa95d8b..679443de 100644 --- a/flake.lock +++ b/flake.lock @@ -43,6 +43,26 @@ "type": "github" } }, + "blockperf": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1758832274, + "narHash": "sha256-na9CqAbZKVQLgxAwyxLOi2hlbQcRln3wgo5qTlBPefo=", + "owner": "johnalotoski", + "repo": "blockperf", + "rev": "80c13b6b3cb6a3a17309851659909bd246e999c8", + "type": "github" + }, + "original": { + "owner": "johnalotoski", + "ref": "jl/fix-detail", + "repo": "blockperf", + "type": "github" + } + }, "blst": { "flake": false, "locked": { @@ -79,11 +99,11 @@ }, "capkgs": { "locked": { - "lastModified": 1755190706, - "narHash": "sha256-/+6TQESSFiqoyT6LfBbcij0HeyCAMD9+AndDJHGKSFM=", + "lastModified": 1763153457, + "narHash": "sha256-2orZuWq9ewAs8RT+kvSeqcbTul4mGaxLCFHYRIUvLvA=", "owner": "input-output-hk", "repo": "capkgs", - "rev": "b2cd2938011557e0c7e767dc112510acdaed2b16", + "rev": "4146aa28984f63ef9cd8fbda904ce345958e23f9", "type": "github" }, "original": { @@ -195,17 +215,17 @@ "cardano-node-service-ng": { "flake": false, "locked": { - "lastModified": 1753292803, - "narHash": "sha256-uP5B+vu/DsqknOAG9EutRSgZW5aAuMV9+NVY+tGJJCk=", + "lastModified": 1763122005, + "narHash": "sha256-pm+lbEiRdQesnkaXmzn58aWlBhD29l7QHGNtJiDlzuA=", "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "f11e0f303ddf3e5b8975daf72ceaa522ddb98426", + "rev": "f5ac0eb01b56af80e8d430828ff6000b6abb92e9", "type": "github" }, "original": { "owner": "IntersectMBO", + "ref": "10.6.0", "repo": "cardano-node", - "rev": "f11e0f303ddf3e5b8975daf72ceaa522ddb98426", "type": "github" } }, @@ -246,17 +266,17 @@ "cardano-submit-api-service-ng": { "flake": false, "locked": { - "lastModified": 1753292803, - "narHash": "sha256-uP5B+vu/DsqknOAG9EutRSgZW5aAuMV9+NVY+tGJJCk=", + "lastModified": 1763122005, + "narHash": "sha256-pm+lbEiRdQesnkaXmzn58aWlBhD29l7QHGNtJiDlzuA=", "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "f11e0f303ddf3e5b8975daf72ceaa522ddb98426", + "rev": "f5ac0eb01b56af80e8d430828ff6000b6abb92e9", "type": "github" }, "original": { "owner": "IntersectMBO", + "ref": "10.6.0", "repo": "cardano-node", - "rev": "f11e0f303ddf3e5b8975daf72ceaa522ddb98426", "type": "github" } }, @@ -280,17 +300,17 @@ "cardano-tracer-service-ng": { "flake": false, "locked": { - "lastModified": 1753292803, - "narHash": "sha256-uP5B+vu/DsqknOAG9EutRSgZW5aAuMV9+NVY+tGJJCk=", + "lastModified": 1763122005, + "narHash": "sha256-pm+lbEiRdQesnkaXmzn58aWlBhD29l7QHGNtJiDlzuA=", "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "f11e0f303ddf3e5b8975daf72ceaa522ddb98426", + "rev": "f5ac0eb01b56af80e8d430828ff6000b6abb92e9", "type": "github" }, "original": { "owner": "IntersectMBO", + "ref": "10.6.0", "repo": "cardano-node", - "rev": "f11e0f303ddf3e5b8975daf72ceaa522ddb98426", "type": "github" } }, @@ -371,6 +391,24 @@ "inputs": { "nixpkgs-lib": "nixpkgs-lib" }, + "locked": { + "lastModified": 1704982712, + "narHash": "sha256-2Ptt+9h8dczgle2Oo6z5ni5rt/uLMG47UFTR1ry/wgg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "07f6395285469419cf9d078f59b5b49993198c00", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_2" + }, "locked": { "lastModified": 1743550720, "narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=", @@ -385,7 +423,7 @@ "type": "github" } }, - "flake-parts_2": { + "flake-parts_3": { "inputs": { "nixpkgs-lib": [ "nix", @@ -496,7 +534,7 @@ "iohk-nix": { "inputs": { "blst": "blst", - "nixpkgs": "nixpkgs", + "nixpkgs": "nixpkgs_2", "secp256k1": "secp256k1", "sodium": "sodium" }, @@ -517,16 +555,16 @@ "iohk-nix-ng": { "inputs": { "blst": "blst_2", - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs_3", "secp256k1": "secp256k1_2", "sodium": "sodium_2" }, "locked": { - "lastModified": 1751421193, - "narHash": "sha256-rklXDo12dfukaSqcEyiYbze3ffRtTl2/WAAQCWfkGiw=", + "lastModified": 1762970280, + "narHash": "sha256-1OZsxij29cBXBFK2NtexgBXbkt2a2sqnRq1HB8RejxE=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "64ca6f4c0c6db283e2ec457c775bce75173fb319", + "rev": "a704b93ea51ee1a8a7e456659e0b28ddba280a95", "type": "github" }, "original": { @@ -538,9 +576,9 @@ "nix": { "inputs": { "flake-compat": "flake-compat_2", - "flake-parts": "flake-parts_2", + "flake-parts": "flake-parts_3", "git-hooks-nix": "git-hooks-nix", - "nixpkgs": "nixpkgs_3", + "nixpkgs": "nixpkgs_4", "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" }, @@ -582,16 +620,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1751071626, - "narHash": "sha256-/uHE/AD2qGq4QLigWAnBHiVvpVXB04XAfrOtw8JMv+Y=", - "owner": "nixos", + "lastModified": 1705458851, + "narHash": "sha256-uQvEhiv33Zj/Pv364dTvnpPwFSptRZgVedDzoM+HqVg=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "a47938d89bdf8e279ad432bd6a473cf4c430f48c", + "rev": "8bf65f17d8070a0a490daf5f1c784b87ee73982c", "type": "github" }, "original": { - "owner": "nixos", - "ref": "release-25.05", + "owner": "NixOS", + "ref": "nixos-23.11", "repo": "nixpkgs", "type": "github" } @@ -613,6 +651,24 @@ } }, "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1703961334, + "narHash": "sha256-M1mV/Cq+pgjk0rt6VxoyyD+O8cOUiai8t9Q6Yyq4noY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b0d36bd0a420ecee3bc916c91886caca87c894e9", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib_2": { "locked": { "lastModified": 1743296961, "narHash": "sha256-b1EdN3cULCqtorQ4QeWgLMrd5ZGOjLSLemfa00heasc=", @@ -676,6 +732,22 @@ } }, "nixpkgs_3": { + "locked": { + "lastModified": 1751071626, + "narHash": "sha256-/uHE/AD2qGq4QLigWAnBHiVvpVXB04XAfrOtw8JMv+Y=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a47938d89bdf8e279ad432bd6a473cf4c430f48c", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "release-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { "locked": { "lastModified": 1747179050, "narHash": "sha256-qhFMmDkeJX9KJwr5H32f1r7Prs7XbQWtO0h3V0a0rFY=", @@ -691,7 +763,7 @@ "type": "github" } }, - "nixpkgs_4": { + "nixpkgs_5": { "locked": { "lastModified": 1748162331, "narHash": "sha256-rqc2RKYTxP3tbjA+PB3VMRQNnjesrT0pEofXQTrMsS8=", @@ -742,6 +814,7 @@ "inputs": { "auth-keys-hub": "auth-keys-hub", "blockfrost-platform-service": "blockfrost-platform-service", + "blockperf": "blockperf", "capkgs": "capkgs", "cardano-db-sync-schema": "cardano-db-sync-schema", "cardano-db-sync-schema-ng": "cardano-db-sync-schema-ng", @@ -757,12 +830,12 @@ "cardano-tracer-service-ng": "cardano-tracer-service-ng", "cardano-wallet-service": "cardano-wallet-service", "colmena": "colmena", - "flake-parts": "flake-parts", + "flake-parts": "flake-parts_2", "inputs-check": "inputs-check", "iohk-nix": "iohk-nix", "iohk-nix-ng": "iohk-nix-ng", "nix": "nix", - "nixpkgs": "nixpkgs_4", + "nixpkgs": "nixpkgs_5", "nixpkgs-unstable": "nixpkgs-unstable", "opentofu-registry": "opentofu-registry", "process-compose-flake": "process-compose-flake", diff --git a/flake.nix b/flake.nix index 3d79c2f7..17155394 100644 --- a/flake.nix +++ b/flake.nix @@ -60,9 +60,14 @@ # Cardano related inputs capkgs.url = "github:input-output-hk/capkgs"; iohk-nix.url = "github:input-output-hk/iohk-nix"; - # iohk-nix.url = "path:/home/jlotoski/work/iohk/iohk-nix-wt/iohk-nix"; iohk-nix-ng.url = "github:input-output-hk/iohk-nix"; - # iohk-nix-ng.url = "path:/home/jlotoski/work/iohk/iohk-nix-wt/iohk-nix"; + # iohk-nix-ng.url = "path:/home/jlotoski/work/iohk/iohk-nix-wt/jl/10.6.0-pre-updates"; + + # Until blockperf detail fix is merged to master upstream + blockperf.url = "github:johnalotoski/blockperf/jl/fix-detail"; + + # Until cardano-node 10.6.0 is tagged + # cardano-node-10-6-0.url = "github:IntersectMBO/cardano-node"; # For tmp local testing pins # cardano-faucet.url = "github:input-output-hk/cardano-faucet/jl/node-9.2"; @@ -96,14 +101,12 @@ }; cardano-node-service = { - # Until 10.6.0 tag is available url = "github:IntersectMBO/cardano-node/f11e0f303ddf3e5b8975daf72ceaa522ddb98426"; flake = false; }; cardano-node-service-ng = { - # Until 10.6.0 tag is available - url = "github:IntersectMBO/cardano-node/f11e0f303ddf3e5b8975daf72ceaa522ddb98426"; + url = "github:IntersectMBO/cardano-node/10.6.0"; flake = false; }; @@ -123,26 +126,22 @@ }; cardano-submit-api-service = { - # Until 10.6.0 tag is available url = "github:IntersectMBO/cardano-node/f11e0f303ddf3e5b8975daf72ceaa522ddb98426"; flake = false; }; cardano-submit-api-service-ng = { - # Until 10.6.0 tag is available - url = "github:IntersectMBO/cardano-node/f11e0f303ddf3e5b8975daf72ceaa522ddb98426"; + url = "github:IntersectMBO/cardano-node/10.6.0"; flake = false; }; cardano-tracer-service = { - # Until 10.6.0 tag is available url = "github:IntersectMBO/cardano-node/f11e0f303ddf3e5b8975daf72ceaa522ddb98426"; flake = false; }; cardano-tracer-service-ng = { - # Until 10.6.0 tag is available - url = "github:IntersectMBO/cardano-node/f11e0f303ddf3e5b8975daf72ceaa522ddb98426"; + url = "github:IntersectMBO/cardano-node/10.6.0"; flake = false; }; diff --git a/flake/nixosModules/profile-basic.nix b/flake/nixosModules/profile-basic.nix index e274fb77..628ef1e9 100644 --- a/flake/nixosModules/profile-basic.nix +++ b/flake/nixosModules/profile-basic.nix @@ -50,12 +50,37 @@ }; environment = { + shellAliases = { + # Remote to remote rsync over ssm is likely to be bandwidth limited + # due to proxying. Consider using wush instead. An alias is used in + # preference over bash function for better discoverability. + # + # Example usage, where the hostname in the rsync args is the EC2 ID: + # rsync-ssm "$REGION" "$RSYNC_ARGS[@]" + rsync-ssm = '' + rsyncSsm(){ + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_SESSION_TOKEN" ]; then + echo "If needed, short term credentials from the deployer can be generated with \"just aws-sso-export\" and sourced on this machine." + fi + REGION="$1" + ARGS="''${@:2}" + rsync -e "ssh -o $(ssm-proxy-rsync $REGION)" ''${ARGS[@]}; + } + rsyncSsm "$@" \ + ''; + }; + + # These can be discovered with `declare -F` for bash function name only and `declare -f` for code shellInit = '' - # This can be used to simplify ssh sessions, rsync, ex: - # ssh -o "$(ssm-proxy-cmd "$REGION")" "$INSTANCE_ID" + # This can be used to simplify ssh sessions, ex: `ssh -o "$(ssm-proxy-cmd "$REGION")" "$INSTANCE_ID"` ssm-proxy-cmd() { echo "ProxyCommand=sh -c 'aws --region $1 ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p'" } + + # This is used to simplify rsync sessions with the rsync-ssm alias: `rsync-ssm "$REGION" "$RSYNC_ARGS[@]"` + ssm-proxy-rsync () { + echo "ProxyCommand='sh -c \"aws --region $1 ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p\"'" + } ''; systemPackages = with pkgs; [ @@ -100,6 +125,7 @@ tcpdump tree wget + wush ]; }; diff --git a/flake/nixosModules/profile-blockperf.nix b/flake/nixosModules/profile-blockperf.nix index efdae7ba..f0c49305 100644 --- a/flake/nixosModules/profile-blockperf.nix +++ b/flake/nixosModules/profile-blockperf.nix @@ -275,27 +275,27 @@ flake: { else { TraceOptions = { "BlockFetch.Client.CompletedBlockFetch" = { - details = "DNormal"; + detail = "DNormal"; maxFrequency = 0.0; severity = "Info"; }; "BlockFetch.Client.SendFetchRequest" = { - details = "DNormal"; + detail = "DNormal"; maxFrequency = 0.0; severity = "Info"; }; "ChainDB.AddBlockEvent.AddedToCurrentChain" = { - details = "DNormal"; + detail = "DNormal"; maxFrequency = 0.0; severity = "Info"; }; "ChainDB.AddBlockEvent.SwitchedToAFork" = { - details = "DNormal"; + detail = "DNormal"; maxFrequency = 0.0; severity = "Info"; }; "ChainSync.Client.DownloadedHeader" = { - details = "DNormal"; + detail = "DNormal"; maxFrequency = 0.0; severity = "Info"; }; diff --git a/flake/nixosModules/profile-cardano-node-group.nix b/flake/nixosModules/profile-cardano-node-group.nix index 67b52109..58ddf9f8 100644 --- a/flake/nixosModules/profile-cardano-node-group.nix +++ b/flake/nixosModules/profile-cardano-node-group.nix @@ -37,8 +37,8 @@ nodeResources, ... }: let - inherit (builtins) elem fromJSON readFile; - inherit (lib) boolToString concatStringsSep flatten foldl' getExe min mkDefault mkForce mkIf mkOption optional optionalAttrs optionalString range recursiveUpdate types; + inherit (builtins) elem fromJSON isString readFile; + inherit (lib) boolToString concatStringsSep flatten foldl' getExe min mkDefault mkForce mkIf mkOption optional optionalAttrs optionalString range recursiveUpdate types versionAtLeast; inherit (types) bool float ints listOf oneOf str; inherit (nodeResources) cpuCount memMiB; @@ -47,11 +47,17 @@ inherit (nixos.config.cardano-parts.perNode.meta) cardanoNodePort cardanoNodePrometheusExporterPort hostAddr hostAddrIpv6 nodeId; inherit (nixos.config.cardano-parts.perNode.pkgs) cardano-cli cardano-node cardano-node-pkgs cardano-tracer mithril-client-cli; inherit (cardanoLib) mkEdgeTopology mkEdgeTopologyP2P; - inherit (cardanoLib.environments.${environmentName}.nodeConfig) ByronGenesisFile ShelleyGenesisFile; + inherit (cardanoLib.environments.${environmentName}.${nodeConfigGenesis}) ByronGenesisFile ShelleyGenesisFile; inherit (opsLib) mithrilAllowedAncillaryNetworks mithrilAllowedNetworks mithrilVerifyingPools; inherit ((fromJSON (readFile ByronGenesisFile)).protocolConsts) protocolMagic; inherit (fromJSON (readFile ShelleyGenesisFile)) slotsPerKESPeriod; + # Remove this usage once legacy tracing is dropped from node + nodeConfigGenesis = + if cardanoLib.environments.${environmentName} ? nodeConfig + then "nodeConfig" + else "nodeConfigLegacy"; + opsLib = self.cardano-parts.lib.opsLib pkgs; numCeil = num: ceiling: @@ -85,7 +91,7 @@ inherit (env) edgeNodes useLedgerAfterSlot; }; in - if cfgNode.useNewTopology + if (elem cfgNode.useNewTopology [null true]) then p2pTopology else legacyTopology; @@ -96,6 +102,8 @@ cfgNode = nixos.config.services.cardano-node; cfgMithril = nixos.config.services.mithril-client; cfgTracer = nixos.config.services.cardano-tracer; + + optNode = nixos.options.services.cardano-node; in { key = ./profile-cardano-node-group.nix; @@ -277,8 +285,17 @@ ); # Fall back to the iohk-nix environment base topology definition if no custom producers are defined. - useNewTopology = mkDefault true; + # As of cardano-node version 10.6.0, useNewTopology is deprecated and will be removed upon Dijkstra hard fork. + # Once 10.6.0 becomes a full release and not only a pre-release, the if statement here can be simplified to only set the null case. + # Once ouroboros-network >= 0.22.2 is merged into node with 10.6.0, useNewTopology should be set to null. + useNewTopology = + if optNode.useNewTopology.type.description == "boolean" + then mkDefault true + # When ouroboros-network >= 0.22.2 is in use: + else mkDefault null; + useSystemdReload = mkDefault true; + topology = mkDefault ( if (cfgNode.producers == []) @@ -290,7 +307,12 @@ else null ); - tracerSocketPathConnect = mkIf (!cfgNode.useLegacyTracing) (mkDefault cfgTracer.acceptingSocket); + # Once 10.6.0 becomes a full release and not only a pre-release, the if statement here can be simplified to only set the acceptAt case. + tracerSocketPathConnect = mkIf (!cfgNode.useLegacyTracing) ( + if cfgTracer ? acceptAt + then mkDefault cfgTracer.acceptAt + else mkDefault cfgTracer.acceptingSocket + ); hostAddr = mkDefault hostAddr; @@ -323,6 +345,14 @@ ]; defaultScribes = [["JournalSK" "cardano"]]; + } + # Remove this optionalAttrs block once 10.6.0 is the latest full release + // optionalAttrs ( + !(versionAtLeast cfgNode.nodeConfig.MinNodeVersion "10.6.0") && (isString cfgNode.operationalCertificate) + ) { + PeerSharing = false; + TargetNumberOfKnownPeers = 100; + TargetNumberOfRootPeers = 100; }; extraNodeInstanceConfig = i: diff --git a/flake/nixosModules/profile-cardano-node-topology.nix b/flake/nixosModules/profile-cardano-node-topology.nix index 299e1972..854353ce 100644 --- a/flake/nixosModules/profile-cardano-node-topology.nix +++ b/flake/nixosModules/profile-cardano-node-topology.nix @@ -95,11 +95,6 @@ }; # These are also set from the role-block-producer nixos module - extraNodeConfig = { - PeerSharing = false; - TargetNumberOfKnownPeers = 100; - TargetNumberOfRootPeers = 100; - }; publicProducers = mkForce (extraNodeListPublicProducers ++ extraPublicProducers); useLedgerAfterSlot = -1; }; @@ -376,8 +371,6 @@ config = { services.cardano-node = { - extraNodeConfig = mkIf (cfg.role == "bp") roles.${cfg.role}.extraNodeConfig; - bootstrapPeers = mkIf (cfg.role == "bp") null; producers = mkIf (cfg.role != null || cfg.enableProducers) ( diff --git a/flake/nixosModules/role-block-producer.nix b/flake/nixosModules/role-block-producer.nix index 992ab0d4..363f0645 100644 --- a/flake/nixosModules/role-block-producer.nix +++ b/flake/nixosModules/role-block-producer.nix @@ -59,9 +59,9 @@ flake: { inherit groupOutPath groupName name secretName keyName pathPrefix; fileOwner = "cardano-node"; fileGroup = "cardano-node"; - reloadUnits = optionals (nodeCfg.useSystemdReload && nodeCfg.useNewTopology) ["cardano-node.service"]; + reloadUnits = optionals (nodeCfg.useSystemdReload && (elem nodeCfg.useNewTopology [null true])) ["cardano-node.service"]; restartUnits = - optionals (!nodeCfg.useSystemdReload || !nodeCfg.useNewTopology) ["cardano-node.service"] + optionals (!nodeCfg.useSystemdReload || !(elem nodeCfg.useNewTopology [null true])) ["cardano-node.service"] ++ optionals mithrilCfg.enable ["mithril-signer.service"]; }; @@ -193,8 +193,6 @@ flake: { serviceCfg.${Protocol} // { # These are also set from the profile-cardano-node-topology nixos module when role == "bp" - extraNodeConfig.PeerSharing = false; - extraNodeConfig.TargetNumberOfRootPeers = 100; publicProducers = mkForce []; useLedgerAfterSlot = -1; }; diff --git a/flakeModules/cluster.nix b/flakeModules/cluster.nix index eaa05f94..f1bdfd77 100644 --- a/flakeModules/cluster.nix +++ b/flakeModules/cluster.nix @@ -11,8 +11,12 @@ # flake.cardano-parts.cluster.infra.aws.region # flake.cardano-parts.cluster.infra.aws.regions # flake.cardano-parts.cluster.infra.generic.abortOnMissingIpModule +# flake.cardano-parts.cluster.infra.generic.costCenter +# flake.cardano-parts.cluster.infra.generic.environment # flake.cardano-parts.cluster.infra.generic.function # flake.cardano-parts.cluster.infra.generic.organization +# flake.cardano-parts.cluster.infra.generic.owner +# flake.cardano-parts.cluster.infra.generic.project # flake.cardano-parts.cluster.infra.generic.repo # flake.cardano-parts.cluster.infra.generic.tribe # flake.cardano-parts.cluster.infra.generic.warnOnMissingIpModule @@ -223,6 +227,35 @@ flake @ { default = true; }; + costCenter = mkOption { + type = optionCheck "string" "infra.generic.costCenter" "str"; + description = mdDoc '' + The cardano-parts cluster infra generic costCenter. + + By default, the costCenter string for tagging cloud resources is + treated as a secret and must be added to the + secrets/tf/cluster.tfvars file. + + The string declared for this option should be the name of the secret + var stored in the cluster.tfvars file. + + This option is required by IOG IT/Finance. + ''; + example = "tag_costCenter"; + default = null; + }; + + environment = mkOption { + type = optionCheck "string" "infra.generic.environment" "str"; + description = mdDoc '' + The cardano-parts cluster infra generic environment. + + This option is required by IOG IT/Finance. + ''; + example = "testnets"; + default = null; + }; + function = mkOption { type = optionCheck "string" "infra.generic.function" "str"; description = mdDoc "The cardano-parts cluster infra generic function."; @@ -237,6 +270,28 @@ flake @ { default = null; }; + owner = mkOption { + type = optionCheck "string" "infra.generic.owner" "str"; + description = mdDoc '' + The cardano-parts cluster infra generic owner. + + This option is required by IOG IT/Finance. + ''; + example = "ioe"; + default = null; + }; + + project = mkOption { + type = optionCheck "string" "infra.generic.project" "str"; + description = mdDoc '' + The cardano-parts cluster infra generic project. + + This option is required by IOG IT/Finance. + ''; + example = "cardano-playground"; + default = null; + }; + repo = mkOption { type = optionCheck "string" "infra.generic.repo" "str"; description = mdDoc "The cardano-parts cluster infra generic repo."; diff --git a/flakeModules/entrypoints.nix b/flakeModules/entrypoints.nix index f4175b49..ee4b81bf 100644 --- a/flakeModules/entrypoints.nix +++ b/flakeModules/entrypoints.nix @@ -41,9 +41,13 @@ in { cp -r "${envCfgs}/config/$ENV" "$DATA_DIR/config/" done + # Required for direct entrypoint access. + # Can be removed once the find command below is removed + chmod -R +w "$DATA_DIR" + # Until https://github.com/IntersectMBO/cardano-node/pull/6282 is merged and released, # remove PrometheusSimple from configs so multiple instances can be started without fatal error. - find "$DATA_DIR/config" -type f -exec sed -i '/PrometheusSimple/d' {} + + find "$DATA_DIR" -name 'config*.json' -type f -exec sed -i '/PrometheusSimple/d' {} + # Prepare mithril client env configs ${ @@ -88,6 +92,7 @@ in { [ -z "''${DATA_DIR:-}" ] && echo "DATA_DIR env var must be set -- aborting" && exit 1 mkdir -p "$DATA_DIR/config/custom" + chmod -R +w "$DATA_DIR/config" # The menu of environments that we ship as built-in envs if [ "''${UNSTABLE_LIB:-}" = "true" ]; then @@ -98,7 +103,7 @@ in { ${copyEnvsTemplate cardanoLib} fi - chmod -R +w "$DATA_DIR/config" "$DATA_DIR/config/custom" + chmod -R +w "$DATA_DIR/config" if [ -n "''${ENVIRONMENT:-}" ]; then echo "Using the preset environment $ENVIRONMENT ..." >&2 diff --git a/flakeModules/jobs.nix b/flakeModules/jobs.nix index 9a7f62d8..0438fbf5 100644 --- a/flakeModules/jobs.nix +++ b/flakeModules/jobs.nix @@ -258,6 +258,7 @@ in { # [$TEMPLATE_DIR] # [$TESTNET_MAGIC] # [$UNSTABLE] + # [$UNSTABLE_LIB] # [$USE_ENCRYPTION] # [$USE_SHELL_BINS] @@ -275,6 +276,7 @@ in { if [ "''${UNSTABLE_LIB:-}" = "true" ]; then export TEMPLATE_DIR=''${TEMPLATE_DIR:-"${localFlake.inputs.iohk-nix-ng}/cardano-lib/testnet-template"} + echo "$TEMPLATE_DIR" else export TEMPLATE_DIR=''${TEMPLATE_DIR:-"${localFlake.inputs.iohk-nix}/cardano-lib/testnet-template"} fi @@ -285,7 +287,7 @@ in { ENV="''${ENV:-custom}" mkdir -p "$GENESIS_DIR" - "''${CARDANO_CLI_NO_ERA[@]}" legacy genesis create-cardano \ + "''${CARDANO_CLI_NO_ERA[@]}" legacy genesis create-cardano --conway-era \ --genesis-dir "$GENESIS_DIR" \ --gen-genesis-keys "$NUM_GENESIS_KEYS" \ --gen-utxo-keys 1 \ diff --git a/flakeModules/lib.nix b/flakeModules/lib.nix index 1fba2bab..287eb52d 100644 --- a/flakeModules/lib.nix +++ b/flakeModules/lib.nix @@ -3,15 +3,12 @@ # TODO: Move this to a docs generator # # Attributes available on flakeModule import: +# flake.cardano-parts.lib.opsLib # flake.cardano-parts.lib.topologyLib # # Tips: # * flake level attrs are accessed from flake level at [config.]flake.cardano-parts.lib.<...> -{ - config, - lib, - ... -}: let +{lib, ...}: let inherit (lib) mdDoc mkDefault mkOption types; inherit (types) attrsOf anything functionTo submodule; diff --git a/flakeModules/lib/ops.nix b/flakeModules/lib/ops.nix index 3187af8f..f3b7802d 100644 --- a/flakeModules/lib/ops.nix +++ b/flakeModules/lib/ops.nix @@ -21,13 +21,17 @@ with lib; rec { cp -v "${cardano-deployment}/$i" "$out/config/$ENV/''${i#"$ENV-"}" done - # Adjust genesis file, config and config-bp refs - for i in config config-bp db-sync-config; do + # Adjust genesis file, config refs + for i in config config-legacy db-sync-config; do if [ -f "$out/config/$ENV/$i.json" ]; then sed -i "s|\"$ENV-|\"|g" "$out/config/$ENV/$i.json" fi done + # Normalize the topology file peer snapshot ref for per ENV dir placement + ${jq}/bin/jq '.peerSnapshotFile = "peer-snapshot.json"' < "$out/config/$ENV/topology.json" > "$ENV-topology.json" + mv -v "$ENV-topology.json" "$out/config/$ENV/topology.json" + # Adjust index.html file refs sed -i "s|$ENV-|config/$ENV/|g" "$out/index.html" done diff --git a/flakeModules/pkgs.nix b/flakeModules/pkgs.nix index b0378f74..0e98eef8 100644 --- a/flakeModules/pkgs.nix +++ b/flakeModules/pkgs.nix @@ -464,20 +464,25 @@ in dbsync-release = "input-output-hk-cardano-db-sync-13-6-0-5-cb61094"; dbsync-pre-release = "input-output-hk-cardano-db-sync-13-6-0-5-cb61094"; metadata-server-release = "input-output-hk-offchain-metadata-tools-ops-1-0-0-f406c6d"; - mithril-release = "input-output-hk-mithril-2524-0-pre-7bf7033"; - mithril-pre-release = "input-output-hk-mithril-unstable-de4de82"; + mithril-release = "input-output-hk-mithril-2543-1-hotfix-5d5571e"; + mithril-pre-release = "input-output-hk-mithril-unstable-59bf453"; node-release = "input-output-hk-cardano-node-10-5-1-ca1ec27"; - node-pre-release = "input-output-hk-cardano-node-10-5-1-ca1ec27"; + node-pre-release = "input-output-hk-cardano-node-10-6-0-f5ac0eb"; in submodule { options = foldl' recursiveUpdate {} [ # TODO: Fix the missing meta/version info upstream (mkPkg "bech32" caPkgs."bech32-${node-release}") (mkPkg "blockfrost-platform" caPkgs.default-blockfrost-blockfrost-platform-0-0-2-e06029b) - (mkPkg "blockperf" caPkgs.blockperf-cardano-foundation-blockperf-main-d757f38) + + # Until blockperf detail fix is merged to master upstream + # (mkPkg "blockperf" caPkgs.blockperf-cardano-foundation-blockperf-main-d757f38) + (mkPkg "blockperf" localFlake.inputs.blockperf.packages.x86_64-linux.blockperf) + (mkPkg "cardano-address" caPkgs."\"cardano-addresses:exe:cardano-address\"-IntersectMBO-cardano-addresses-4-0-0-3749045") (mkPkg "cardano-cli" (caPkgs."cardano-cli-${node-release}" // {version = "10.11.0.0";})) - (mkPkg "cardano-cli-ng" (caPkgs."cardano-cli-${node-pre-release}" // {version = "10.11.0.0";})) + (mkPkg "cardano-cli-ng" (caPkgs."cardano-cli-${node-pre-release}" // {version = "10.13.1.0";})) + # (mkPkg "cardano-cli-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.cardano-cli) (mkPkg "cardano-db-sync" caPkgs."\"cardano-db-sync:exe:cardano-db-sync\"-${dbsync-release}") (mkPkg "cardano-db-sync-ng" caPkgs."\"cardano-db-sync:exe:cardano-db-sync\"-${dbsync-pre-release}") (mkPkg "cardano-db-tool" caPkgs."\"cardano-db-tool:exe:cardano-db-tool\"-${dbsync-release}") @@ -490,17 +495,21 @@ in (mkPkg "cardano-faucet-ng" caPkgs."\"cardano-faucet:exe:cardano-faucet\"-input-output-hk-cardano-faucet-10-1-2cccf6d") (mkPkg "cardano-node" (caPkgs."cardano-node-${node-release}" // {version = "10.5.1";})) - (mkPkg "cardano-node-ng" (caPkgs."cardano-node-${node-pre-release}" // {version = "10.5.1";})) + (mkPkg "cardano-node-ng" (caPkgs."cardano-node-${node-pre-release}" // {version = "10.6.0";})) + # (mkPkg "cardano-node-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.cardano-node) (mkPkg "cardano-ogmios" caPkgs.ogmios-input-output-hk-cardano-ogmios-v6-11-2-df5971a) (mkPkg "cardano-signer" caPkgs.cardano-signer-johnalotoski-cardano-signer-v1-29-0-2ef95e1) (mkPkg "cardano-smash" caPkgs."cardano-smash-server-no-basic-auth-${dbsync-release}") (mkPkg "cardano-smash-ng" caPkgs."cardano-smash-server-no-basic-auth-${dbsync-pre-release}") (mkPkg "cardano-submit-api" caPkgs."cardano-submit-api-${node-release}") (mkPkg "cardano-submit-api-ng" caPkgs."cardano-submit-api-${node-pre-release}") + # (mkPkg "cardano-submit-api-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.cardano-submit-api) (mkPkg "cardano-testnet" caPkgs."cardano-testnet-${node-release}") (mkPkg "cardano-testnet-ng" caPkgs."cardano-testnet-${node-pre-release}") + # (mkPkg "cardano-testnet-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.cardano-testnet) (mkPkg "cardano-tracer" caPkgs."cardano-tracer-${node-release}") (mkPkg "cardano-tracer-ng" caPkgs."cardano-tracer-${node-pre-release}") + # (mkPkg "cardano-tracer-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.cardano-tracer) (mkPkg "cardano-wallet" (caPkgs.cardano-wallet-cardano-foundation-cardano-wallet-v2025-03-31-1649791 // { pname = "cardano-wallet"; @@ -510,10 +519,13 @@ in (mkPkg "colmena" localFlake.inputs.colmena.packages.${system}.colmena) (mkPkg "db-analyser" caPkgs."db-analyser-${node-release}") (mkPkg "db-analyser-ng" caPkgs."db-analyser-${node-pre-release}") + # (mkPkg "db-analyser-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.db-analyser) (mkPkg "db-synthesizer" caPkgs."db-synthesizer-${node-release}") (mkPkg "db-synthesizer-ng" caPkgs."db-synthesizer-${node-pre-release}") + # (mkPkg "db-synthesizer-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.db-synthesizer) (mkPkg "db-truncater" caPkgs."db-truncater-${node-release}") (mkPkg "db-truncater-ng" caPkgs."db-truncater-${node-pre-release}") + # (mkPkg "db-truncater-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.db-truncater) (mkPkg "isd" caPkgs.isd-isd-project-isd-v0-5-1-51d52a2) (mkPkg "process-compose" caPkgs.process-compose-F1bonacc1-process-compose-v1-46-0-6a1799e) (mkPkg "metadata-server" caPkgs."metadata-server-${metadata-server-release}") @@ -527,6 +539,7 @@ in (mkPkg "orchestrator-cli" caPkgs."orchestrator-cli-${credential-manager-release}") (mkPkg "snapshot-converter" caPkgs."snapshot-converter-${node-release}") (mkPkg "snapshot-converter-ng" caPkgs."snapshot-converter-${node-pre-release}") + # (mkPkg "snapshot-converter-ng" localFlake.inputs.cardano-node-10-6-0.packages.x86_64-linux.snapshot-converter) (mkPkg "token-metadata-creator" (recursiveUpdate caPkgs."token-metadata-creator-${metadata-server-release}" {meta.mainProgram = "token-metadata-creator";})) (mkPkg "tx-bundle" caPkgs."tx-bundle-${credential-manager-release}") ]; diff --git a/flakeModules/shell.nix b/flakeModules/shell.nix index b37584d9..b320f94d 100644 --- a/flakeModules/shell.nix +++ b/flakeModules/shell.nix @@ -11,10 +11,12 @@ # perSystem.cardano-parts.shell.>.defaultHooks # perSystem.cardano-parts.shell.>.defaultLintPkg # perSystem.cardano-parts.shell.>.defaultVars +# perSystem.cardano-parts.shell.>.defaultZshCompFpathLoading # perSystem.cardano-parts.shell.>.enableFormatter # perSystem.cardano-parts.shell.>.enableHooks # perSystem.cardano-parts.shell.>.enableLint # perSystem.cardano-parts.shell.>.enableVars +# perSystem.cardano-parts.shell.>.enableZshCompFpathLoading # perSystem.cardano-parts.shell.>.extraPkgs # perSystem.cardano-parts.shell.>.pkgs # @@ -41,7 +43,7 @@ ... }: let inherit (flake-parts-lib) mkPerSystemOption; - inherit (lib.types) anything attrs attrsOf bool enum nullOr listOf package str submodule; + inherit (lib.types) anything attrs attrsOf bool enum nullOr listOf package str lines submodule; in with builtins; with lib; { @@ -112,7 +114,7 @@ in }; defaultHooks = mkOption { - type = globalType isGlobal str; + type = globalType isGlobal lines; description = mdDoc "The cardano-parts default git and shell hooks."; default = globalDefault isGlobal '' if ${isPartsRepo} && [ -d .git/hooks ]; then @@ -152,6 +154,48 @@ in }; }; + defaultZshCompFpathLoading = mkOption { + description = mdDoc "The cardano-parts default zsh completion fpath loading hook."; + default = globalDefault isGlobal (packages: '' + # Direnv use prevents the dynamic loading of zsh completions, + # requiring a zsh "reentry" upon arriving in the devShell. + # (ie: `zsh` or `exec zsh -l`, etc. + # + # If using plain nix develop, zsh "reentry" is not required. + export ZDOTDIR=$PWD/.direnv-zsh + mkdir -p "$ZDOTDIR" + rm -f "$ZDOTDIR/.zcompdump"* + + cat > "$ZDOTDIR/completions.zsh" <<'EOF' + for p in ${concatStringsSep " " (map (p: "${p}") packages)}; do + if [ -d "$p/share/zsh/site-functions" ]; then + fpath=("$p/share/zsh/site-functions" $fpath) + fi + done + autoload -U compinit + compinit + EOF + + cat > "$ZDOTDIR/.zshrc" <<'EOF' + if [ -f "$HOME/.zshrc" ]; then + source "$HOME/.zshrc" + fi + + source "$ZDOTDIR/completions.zsh" + autoload -U compinit + compinit -C + + alias zsh-base='unset ZDOTDIR; exec zsh -l' + echo + echo "To return to your normal zsh without devShell fpath modifications, run \"zsh-base\" before leaving the repo directory," + echo "otherwise, close and open a new zsh shell to avoid lingering devShell command completions." + echo + echo "To re-enter the completions in this devShell with direnv, run \"zsh\" or similar." + echo + EOF + ''); + }; + enableFormatter = mkOption { type = globalType isGlobal bool; description = mdDoc "Enable default cardano-parts formatter in the devShells."; @@ -176,6 +220,12 @@ in default = globalDefault isGlobal true; }; + enableZshCompFpathLoading = mkOption { + type = globalType isGlobal bool; + description = mdDoc "Enable default cardano-parts zsh completion loading into fpath in zsh devShells."; + default = globalDefault isGlobal true; + }; + extraPkgs = mkOption { type = listOf package; description = mdDoc "Extra packages."; @@ -241,6 +291,12 @@ in description = mdDoc "Minimal devShell"; extraCfg.pkgs = mkOption { default = with pkgs; [ + # Bash interactive needs to be included, otherwise, the + # default devShell uses a bash without progcomp compiled + # and devShell bins will fail to auto-complete on file + # completions. + bashInteractive + alejandra bc curl @@ -421,12 +477,17 @@ in devShells = let mkShell = id: - pkgs.mkShell ({ + pkgs.mkShell (rec { packages = allPkgs id; shellHook = # Add optional git/shell and formatter hooks selectScope id optionalString "enableHooks" "defaultHooks" + selectScope id optionalAttrs "enableFormatter" "defaultFormatterHook" + + (selectScope id (cond: as: + if cond + then as + else _: "") "enableZshCompFpathLoading" "defaultZshCompFpathLoading") + packages + '' [ -z "$NOMENU" ] && menu ''; diff --git a/templates/cardano-parts-project/.gitignore b/templates/cardano-parts-project/.gitignore index 7f481205..668323da 100644 --- a/templates/cardano-parts-project/.gitignore +++ b/templates/cardano-parts-project/.gitignore @@ -1,6 +1,6 @@ /*.cert /cloudFormation/ -/.direnv +/.direnv* /.envrc.local /.gcroots /*.log diff --git a/templates/cardano-parts-project/Justfile b/templates/cardano-parts-project/Justfile index 31b6c878..7fa2b88e 100644 --- a/templates/cardano-parts-project/Justfile +++ b/templates/cardano-parts-project/Justfile @@ -284,8 +284,17 @@ cardano-testnet isNg *ARGS: cf STACKNAME: #!/usr/bin/env nu mkdir cloudFormation + let secretName = (nix eval --raw '.#cardano-parts.cluster.infra.generic.costCenter') + let costCenter = ( + just sops-decrypt-binary secrets/tf/cluster.tfvars + | lines + | where { |it| $it =~ $secretName } + | parse $"($secretName) = \"{secret}\"" + | get 0.secret + | to text + ) nix eval --json '.#cloudFormation.{{STACKNAME}}' | from json | save --force 'cloudFormation/{{STACKNAME}}.json' - rain deploy --debug --termination-protection --yes ./cloudFormation/{{STACKNAME}}.json + rain deploy --debug --params costCenter=($costCenter) --termination-protection ./cloudFormation/{{STACKNAME}}.json # Prep dbsync for delegation analysis dbsync-prep ENV HOST ACCTS="501": @@ -401,11 +410,11 @@ dedelegate-pools ENV *IDXS=null: --wallet-mnemonic <(just sops-decrypt-binary secrets/envs/{{ENV}}/utxo-keys/faucet.mnemonic) \ --delegation-index "$i" - TXID=$(eval "$CARDANO_CLI" latest transaction txid --tx-file tx-deleg-account-$i-restore.txsigned) + TXID=$(eval "$CARDANO_CLI" latest transaction txid --tx-file tx-deleg-account-$i-restore.txsigned | jq -r .txhash) EXISTS="true" while [ "$EXISTS" = "true" ]; do - EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists) + EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists || true) if [ "$EXISTS" = "true" ]; then echo "Pool de-delegation index $i tx still exists in the mempool, sleeping 5s: $TXID" else diff --git a/templates/cardano-parts-project/flake/cloudFormation/terraformState.nix b/templates/cardano-parts-project/flake/cloudFormation/terraformState.nix index c9afc619..a5411d00 100644 --- a/templates/cardano-parts-project/flake/cloudFormation/terraformState.nix +++ b/templates/cardano-parts-project/flake/cloudFormation/terraformState.nix @@ -12,23 +12,45 @@ with lib; { Key = n; Value = v; }) { - inherit (config.flake.cardano-parts.cluster.infra.generic) organization tribe function repo; - environment = "generic"; + inherit + (config.flake.cardano-parts.cluster.infra.generic) + environment + function + organization + owner + project + repo + tribe + ; }) ++ [ { Key = "Name"; Value = name; } + { + Key = "costCenter"; + Value = { + Ref = "costCenter"; + }; + } ]; in { AWSTemplateFormatVersion = "2010-09-09"; Description = "Terraform state handling"; + # The costCenter parameter will be passed to the configuration via a secrets file. + # For details, see the just recipe: cf + Parameters = { + costCenter = { + Type = "String"; + Description = "The costCenter tag"; + }; + }; + # Resources here will be created in the AWS_REGION and AWS_PROFILE from your # environment variables. # Execute this using: `just cf terraformState` - Resources = { kmsKey = { Type = "AWS::KMS::Key"; diff --git a/templates/cardano-parts-project/flake/cluster.nix b/templates/cardano-parts-project/flake/cluster.nix index 5aabd32e..dbfc3d36 100644 --- a/templates/cardano-parts-project/flake/cluster.nix +++ b/templates/cardano-parts-project/flake/cluster.nix @@ -35,6 +35,13 @@ # function = "cardano-parts"; # repo = "https://github.com/input-output-hk/UPDATE_ME"; + # owner = "ioe"; + # environment = "testnets"; + # project = "cardano-playground"; + + # This is the tf var secrets name located in secrets/tf/cluster.tfvars + # costCenter = "tag_costCenter"; + # By default abort and warn if the ip-module is missing: # abortOnMissingIpModule = true; # warnOnMissingIpModule = true; diff --git a/templates/cardano-parts-project/flake/colmena.nix b/templates/cardano-parts-project/flake/colmena.nix index daf7d24a..fd22534a 100644 --- a/templates/cardano-parts-project/flake/colmena.nix +++ b/templates/cardano-parts-project/flake/colmena.nix @@ -7,8 +7,6 @@ }: let inherit (config.flake) nixosModules nixosConfigurations; # inherit (config.flake.cardano-parts.cluster.infra.aws) domain regions; - - cfgGeneric = config.flake.cardano-parts.cluster.infra.generic; in with builtins; with lib; { @@ -39,7 +37,8 @@ in # Since all machines are assigned a group, this is a good place to include default aws instance tags aws.instance.tags = { - inherit (cfgGeneric) organization tribe function repo; + # This group environment name will override the + # flake.cluster.infra.generic environment name for aws instances. environment = config.flake.cardano-parts.cluster.groups.${name}.meta.environmentName; group = name; }; diff --git a/templates/cardano-parts-project/flake/opentofu/cluster.nix b/templates/cardano-parts-project/flake/opentofu/cluster.nix index abd98ef4..b1f3ecec 100644 --- a/templates/cardano-parts-project/flake/opentofu/cluster.nix +++ b/templates/cardano-parts-project/flake/opentofu/cluster.nix @@ -138,6 +138,28 @@ with lib; let groupMultivalueDnsAttrs = mkMultivalueDnsAttrs "groupRelayMultivalueDns" groupMultivalueDnsList; mkCustomRoute53Records = import ./cluster/route53.nix-import; + + sensitiveString = { + type = "string"; + sensitive = true; + nullable = false; + }; + + defaultTags = { + inherit + (infra.generic) + environment + function + organization + owner + project + repo + tribe + ; + + # costCenter is saved as a secret + costCenter = "\${var.${infra.generic.costCenter}}"; + }; in { flake.opentofu.cluster = inputs.cardano-parts.inputs.terranix.lib.terranixConfiguration { inherit system; @@ -161,13 +183,19 @@ in { }; }; + variable = { + # costCenter tag should remain secret in public repos + "${infra.generic.costCenter}" = sensitiveString; + }; + provider.aws = forEach (attrNames cluster.regions) (region: { inherit region; alias = underscore region; - default_tags.tags = { - inherit (infra.generic) organization tribe function repo; - environment = "generic"; - }; + + # Default tagging is inconsistent across aws resources, but including + # it may help tag some resources that might have otherwise been + # missed. + default_tags.tags = defaultTags; }); # Common parameters: @@ -324,6 +352,8 @@ in { gateway_id = "\${data.aws_internet_gateway.${region}.id}"; } ]; + + tags = defaultTags; }; } ); @@ -346,6 +376,7 @@ in { + " cidrsubnet(${ipv6CidrBlock}, ${toString ipv6SubnetCidrBits} - parseint(tolist(regex(\"/([0-9]+)$\", ${ipv6CidrBlock}))[0], 10), each.key)}"; availability_zone = "\${each.value.availability_zone}"; + tags = defaultTags; }; }); @@ -358,6 +389,7 @@ in { ${region} = { provider = awsProviderFor region; assign_generated_ipv6_cidr_block = true; + tags = defaultTags; }; } ); @@ -378,6 +410,10 @@ in { vpc_security_group_ids = [ "\${aws_security_group.common_${underscore region}[0].id}" ]; + + # Provider level `default_tags` are automatically inherited at + # the instance level. Instance specific tags defined in + # flake/colmena.nix are merged. tags = {Name = name;} // node.aws.instance.tags or {}; root_block_device = { @@ -386,14 +422,9 @@ in { iops = node.aws.instance.root_block_device.iops or 3000; throughput = node.aws.instance.root_block_device.throughput or 125; delete_on_termination = true; - tags = - # Root block device tags aren't applied like the other - # resources since terraform-aws-provider v5.39.0. - # - # We need to strip the following tag attrs or tofu - # constantly tries to re-apply them. - {Name = name;} - // removeAttrs (node.aws.instance.tags or {}) ["organization" "tribe" "function" "repo"]; + + # Default tags are not inherited to the volume level automatically. + tags = defaultTags // {Name = name;} // node.aws.instance.tags or {}; }; metadata_options = { @@ -433,6 +464,7 @@ in { aws_iam_instance_profile.ec2_profile = { name = "ec2Profile"; role = "\${aws_iam_role.ec2_role.name}"; + tags = defaultTags; }; aws_iam_role.ec2_role = { @@ -447,6 +479,8 @@ in { } ]; }; + + tags = defaultTags; }; aws_iam_role_policy_attachment = let @@ -495,6 +529,8 @@ in { } ]; }; + + tags = defaultTags; }; tls_private_key.bootstrap.algorithm = "ED25519"; @@ -508,6 +544,7 @@ in { provider = awsProviderFor region; key_name = "bootstrap"; public_key = "\${tls_private_key.bootstrap.public_key_openssh}"; + tags = defaultTags; }; }); @@ -515,6 +552,8 @@ in { inherit (node.aws.instance) count; provider = awsProviderFor node.aws.region; instance = "\${aws_instance.${name}[0].id}"; + + # Provider level `default_tags` are automatically inherited. tags = {Name = name;} // node.aws.instance.tags or {}; }); @@ -592,6 +631,8 @@ in { protocol = "-1"; }) ]; + + tags = defaultTags; }; }); diff --git a/templates/cardano-parts-project/scripts/bash-fns.sh b/templates/cardano-parts-project/scripts/bash-fns.sh index 2096f4ce..409c9916 100644 --- a/templates/cardano-parts-project/scripts/bash-fns.sh +++ b/templates/cardano-parts-project/scripts/bash-fns.sh @@ -2,13 +2,6 @@ # # Various bash helper fns which aren't used enough to move to just recipes. -# This can be used to simplify ssh sessions, rsync, ex: -# ssh -o "$(ssm-proxy-cmd "$REGION")" "$INSTANCE_ID" -ssm-proxy-cmd() { - echo "ProxyCommand=sh -c 'aws --region $1 ssm start-session --target %h --document-name AWS-StartSSHSession --parameters portNumber=%p'" -} - - # A handy transaction submission function with mempool monitoring. # CARDANO_NODE_{NETWORK_ID,SOCKET_PATH}, TESTNET_MAGIC should already be exported. submit() ( @@ -22,7 +15,7 @@ submit() ( EXISTS="true" while [ "$EXISTS" = "true" ]; do - EXISTS=$(cardano-cli latest query tx-mempool tx-exists "$TXID" | jq -re .exists) + EXISTS=$(cardano-cli latest query tx-mempool tx-exists "$TXID" | jq -re .exists || true) if [ "$EXISTS" = "true" ]; then echo "The transaction still exists in the mempool, sleeping 5s: $TXID" else @@ -206,3 +199,30 @@ return-utxo() ( submit "$BASENAME.signed" ) + +# A handy faucet submission function with mempool monitoring, usable on custom networks. +# CARDANO_NODE_{NETWORK_ID,SOCKET_PATH}, TESTNET_MAGIC should already be exported. +# SEND_ADDR, LOVELACE, RICH_ADDR and RICH vars need to be provided. +faucet() ( + SEND_ADDR="$1" + LOVELACE="$2" + + UTXOS=$(cardano-cli query utxo --address "$RICH_ADDR") + UTXO=$(jq -r 'to_entries | max_by(.value.value.lovelace) | { (.key): .value }' <<< "$UTXOS") + UTXO_TX=$(jq -r 'keys[0]' <<< "$UTXO") + + cardano-cli latest transaction build \ + --tx-in "$UTXO_TX" \ + --tx-out "$SEND_ADDR+$LOVELACE" \ + --change-address "$RICH_ADDR" \ + --testnet-magic "$TESTNET_MAGIC" \ + --out-file faucet.txbody + + cardano-cli latest transaction sign \ + --tx-body-file faucet.txbody \ + --signing-key-file "$RICH.skey" \ + --testnet-magic "$TESTNET_MAGIC" \ + --out-file faucet.txsigned + + submit faucet.txsigned +) diff --git a/templates/cardano-parts-project/scripts/lib/cli.py b/templates/cardano-parts-project/scripts/lib/cli.py index 3a434afb..34b8e1aa 100644 --- a/templates/cardano-parts-project/scripts/lib/cli.py +++ b/templates/cardano-parts-project/scripts/lib/cli.py @@ -53,7 +53,7 @@ def createTransaction(start, end, txin, payments_txouts, utxo_address, utxo_sign signed_tx = signTx(tx_prefix, utxo_signing_key_str) - p = subprocess.run([cardanoCliStr(), "latest", "transaction", "txid", "--tx-file", signed_tx], capture_output=True, text=True) + p = subprocess.run([cardanoCliStr(), "latest", "transaction", "txid", "--output-text", "--tx-file", signed_tx], capture_output=True, text=True) new_txin = p.stdout.rstrip() return (f"{new_txin}#0", tx_out_amount, fee) @@ -66,6 +66,7 @@ def estimateFeeTx(txbody, txin_count, txout_count, pparams) -> int: "--tx-out-count", str(txout_count), "--witness-count", "1", "--protocol-params-file", pparams, + "--output-text", "--tx-body-file", txbody] p = subprocess.run(cmd, capture_output=True, text=True) diff --git a/templates/cardano-parts-project/scripts/recipes/governance.just b/templates/cardano-parts-project/scripts/recipes/governance.just index c4c5b488..0206c10c 100644 --- a/templates/cardano-parts-project/scripts/recipes/governance.just +++ b/templates/cardano-parts-project/scripts/recipes/governance.just @@ -992,7 +992,7 @@ vote-with-pool ENV POOL ACTION_ID ACTION_IDX VOTE: EXISTS="true" while [ "$EXISTS" = "true" ]; do - EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists) + EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists || true) if [ "$EXISTS" = "true" ]; then echo "Vote transaction still exists in the mempool, sleeping 5s: $TXID" else diff --git a/templates/cardano-parts-project/scripts/restore-delegation-accounts.py b/templates/cardano-parts-project/scripts/restore-delegation-accounts.py index bebb44ed..1f557810 100755 --- a/templates/cardano-parts-project/scripts/restore-delegation-accounts.py +++ b/templates/cardano-parts-project/scripts/restore-delegation-accounts.py @@ -261,7 +261,7 @@ def signTx(tx_body, utxo_signing_key_str, stake_signing_key, out_file): if p.returncode != 0: print(p.stderr) raise Exception("Unknown error signing transaction") - cli_args = ["cardano-cli", "latest", "transaction", "txid", "--tx-file", out_file] + cli_args = ["cardano-cli", "latest", "transaction", "txid", "--output-text", "--tx-file", out_file] p = subprocess.run(cli_args, input=None, capture_output=True, text=True) if p.returncode != 0: print(p.stderr) diff --git a/templates/cardano-parts-project/secrets/tf/cluster.tfvars b/templates/cardano-parts-project/secrets/tf/cluster.tfvars new file mode 100644 index 00000000..34933f35 --- /dev/null +++ b/templates/cardano-parts-project/secrets/tf/cluster.tfvars @@ -0,0 +1,9 @@ +# To enable terraform and cloudFormation cluster secrets usage: +# * update the following with the appropriate cluster secrets +# * remove these comments +# * encrypt this file with sops as a binary type using an age sre/admin secret key +# +# Expect this file to generate a pre-push error until it is either encrypted or deleted + +# Set this to the finance billing code +tag_costCenter = "UPDATE_ME"