imp: tags update, breaking change fixes, faucet fn addn

Author: johnalotoski

templates/cardano-parts-project/.gitignore

@@ -1,6 +1,6 @@
 /*.cert
 /cloudFormation/
-/.direnv
+/.direnv*
 /.envrc.local
 /.gcroots
 /*.log

templates/cardano-parts-project/Justfile

@@ -284,8 +284,17 @@ cardano-testnet isNg *ARGS:
 cf STACKNAME:
   #!/usr/bin/env nu
   mkdir cloudFormation
+  let secretName = (nix eval --raw '.#cardano-parts.cluster.infra.generic.costCenter')
+  let costCenter = (
+    just sops-decrypt-binary secrets/tf/cluster.tfvars
+    | lines
+    | where { |it| $it =~ $secretName }
+    | parse $"($secretName) = \"{secret}\""
+    | get 0.secret
+    | to text
+  )
   nix eval --json '.#cloudFormation.{{STACKNAME}}' | from json | save --force 'cloudFormation/{{STACKNAME}}.json'
-  rain deploy --debug --termination-protection --yes ./cloudFormation/{{STACKNAME}}.json
+  rain deploy --debug --params costCenter=($costCenter) --termination-protection ./cloudFormation/{{STACKNAME}}.json
 
 # Prep dbsync for delegation analysis
 dbsync-prep ENV HOST ACCTS="501":
@@ -401,11 +410,11 @@ dedelegate-pools ENV *IDXS=null:
       --wallet-mnemonic <(just sops-decrypt-binary secrets/envs/{{ENV}}/utxo-keys/faucet.mnemonic) \
       --delegation-index "$i"
 
-    TXID=$(eval "$CARDANO_CLI" latest transaction txid --tx-file tx-deleg-account-$i-restore.txsigned)
+    TXID=$(eval "$CARDANO_CLI" latest transaction txid --tx-file tx-deleg-account-$i-restore.txsigned | jq -r .txhash)
     EXISTS="true"
 
     while [ "$EXISTS" = "true" ]; do
-      EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists)
+      EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists || true)
       if [ "$EXISTS" = "true" ]; then
         echo "Pool de-delegation index $i tx still exists in the mempool, sleeping 5s: $TXID"
       else

templates/cardano-parts-project/flake/cloudFormation/terraformState.nix

@@ -12,23 +12,45 @@ with lib; {
           Key = n;
           Value = v;
         }) {
-          inherit (config.flake.cardano-parts.cluster.infra.generic) organization tribe function repo;
-          environment = "generic";
+          inherit
+            (config.flake.cardano-parts.cluster.infra.generic)
+            environment
+            function
+            organization
+            owner
+            project
+            repo
+            tribe
+            ;
         })
       ++ [
         {
           Key = "Name";
           Value = name;
         }
+        {
+          Key = "costCenter";
+          Value = {
+            Ref = "costCenter";
+          };
+        }
       ];
   in {
     AWSTemplateFormatVersion = "2010-09-09";
     Description = "Terraform state handling";
 
+    # The costCenter parameter will be passed to the configuration via a secrets file.
+    # For details, see the just recipe: cf
+    Parameters = {
+      costCenter = {
+        Type = "String";
+        Description = "The costCenter tag";
+      };
+    };
+
     # Resources here will be created in the AWS_REGION and AWS_PROFILE from your
     # environment variables.
     # Execute this using: `just cf terraformState`
-
     Resources = {
       kmsKey = {
         Type = "AWS::KMS::Key";

templates/cardano-parts-project/flake/cluster.nix

@@ -35,6 +35,13 @@
       # function = "cardano-parts";
       # repo = "https://github.com/input-output-hk/UPDATE_ME";
 
+      # owner = "ioe";
+      # environment = "testnets";
+      # project = "cardano-playground";
+
+      # This is the tf var secrets name located in secrets/tf/cluster.tfvars
+      # costCenter = "tag_costCenter";
+
       # By default abort and warn if the ip-module is missing:
       # abortOnMissingIpModule = true;
       # warnOnMissingIpModule = true;

templates/cardano-parts-project/flake/colmena.nix

@@ -7,8 +7,6 @@
 }: let
   inherit (config.flake) nixosModules nixosConfigurations;
   # inherit (config.flake.cardano-parts.cluster.infra.aws) domain regions;
-
-  cfgGeneric = config.flake.cardano-parts.cluster.infra.generic;
 in
   with builtins;
   with lib; {
@@ -39,7 +37,8 @@ in
 
         # Since all machines are assigned a group, this is a good place to include default aws instance tags
         aws.instance.tags = {
-          inherit (cfgGeneric) organization tribe function repo;
+          # This group environment name will override the
+          # flake.cluster.infra.generic environment name for aws instances.
           environment = config.flake.cardano-parts.cluster.groups.${name}.meta.environmentName;
           group = name;
         };

templates/cardano-parts-project/flake/opentofu/cluster.nix

@@ -138,6 +138,28 @@ with lib; let
   groupMultivalueDnsAttrs = mkMultivalueDnsAttrs "groupRelayMultivalueDns" groupMultivalueDnsList;
 
   mkCustomRoute53Records = import ./cluster/route53.nix-import;
+
+  sensitiveString = {
+    type = "string";
+    sensitive = true;
+    nullable = false;
+  };
+
+  defaultTags = {
+    inherit
+      (infra.generic)
+      environment
+      function
+      organization
+      owner
+      project
+      repo
+      tribe
+      ;
+
+    # costCenter is saved as a secret
+    costCenter = "\${var.${infra.generic.costCenter}}";
+  };
 in {
   flake.opentofu.cluster = inputs.cardano-parts.inputs.terranix.lib.terranixConfiguration {
     inherit system;
@@ -161,13 +183,19 @@ in {
           };
         };
 
+        variable = {
+          # costCenter tag should remain secret in public repos
+          "${infra.generic.costCenter}" = sensitiveString;
+        };
+
         provider.aws = forEach (attrNames cluster.regions) (region: {
           inherit region;
           alias = underscore region;
-          default_tags.tags = {
-            inherit (infra.generic) organization tribe function repo;
-            environment = "generic";
-          };
+
+          # Default tagging is inconsistent across aws resources, but including
+          # it may help tag some resources that might have otherwise been
+          # missed.
+          default_tags.tags = defaultTags;
         });
 
         # Common parameters:
@@ -324,6 +352,8 @@ in {
                         gateway_id = "\${data.aws_internet_gateway.${region}.id}";
                       }
                     ];
+
+                  tags = defaultTags;
                 };
               }
           );
@@ -346,6 +376,7 @@ in {
                   + " cidrsubnet(${ipv6CidrBlock}, ${toString ipv6SubnetCidrBits} - parseint(tolist(regex(\"/([0-9]+)$\", ${ipv6CidrBlock}))[0], 10), each.key)}";
 
                 availability_zone = "\${each.value.availability_zone}";
+                tags = defaultTags;
               };
             });
 
@@ -358,6 +389,7 @@ in {
                 ${region} = {
                   provider = awsProviderFor region;
                   assign_generated_ipv6_cidr_block = true;
+                  tags = defaultTags;
                 };
               }
           );
@@ -378,6 +410,10 @@ in {
                 vpc_security_group_ids = [
                   "\${aws_security_group.common_${underscore region}[0].id}"
                 ];
+
+                # Provider level `default_tags` are automatically inherited at
+                # the instance level.  Instance specific tags defined in
+                # flake/colmena.nix are merged.
                 tags = {Name = name;} // node.aws.instance.tags or {};
 
                 root_block_device = {
@@ -386,14 +422,9 @@ in {
                   iops = node.aws.instance.root_block_device.iops or 3000;
                   throughput = node.aws.instance.root_block_device.throughput or 125;
                   delete_on_termination = true;
-                  tags =
-                    # Root block device tags aren't applied like the other
-                    # resources since terraform-aws-provider v5.39.0.
-                    #
-                    # We need to strip the following tag attrs or tofu
-                    # constantly tries to re-apply them.
-                    {Name = name;}
-                    // removeAttrs (node.aws.instance.tags or {}) ["organization" "tribe" "function" "repo"];
+
+                  # Default tags are not inherited to the volume level automatically.
+                  tags = defaultTags // {Name = name;} // node.aws.instance.tags or {};
                 };
 
                 metadata_options = {
@@ -433,6 +464,7 @@ in {
           aws_iam_instance_profile.ec2_profile = {
             name = "ec2Profile";
             role = "\${aws_iam_role.ec2_role.name}";
+            tags = defaultTags;
           };
 
           aws_iam_role.ec2_role = {
@@ -447,6 +479,8 @@ in {
                 }
               ];
             };
+
+            tags = defaultTags;
           };
 
           aws_iam_role_policy_attachment = let
@@ -495,6 +529,8 @@ in {
                 }
               ];
             };
+
+            tags = defaultTags;
           };
 
           tls_private_key.bootstrap.algorithm = "ED25519";
@@ -508,13 +544,16 @@ in {
               provider = awsProviderFor region;
               key_name = "bootstrap";
               public_key = "\${tls_private_key.bootstrap.public_key_openssh}";
+              tags = defaultTags;
             };
           });
 
           aws_eip = mapNodes (name: node: {
             inherit (node.aws.instance) count;
             provider = awsProviderFor node.aws.region;
             instance = "\${aws_instance.${name}[0].id}";
+
+            # Provider level `default_tags` are automatically inherited.
             tags = {Name = name;} // node.aws.instance.tags or {};
           });
 
@@ -592,6 +631,8 @@ in {
                     protocol = "-1";
                   })
                 ];
+
+                tags = defaultTags;
               };
             });
 

templates/cardano-parts-project/scripts/bash-fns.sh

@@ -15,7 +15,7 @@ submit() (
 
   EXISTS="true"
   while [ "$EXISTS" = "true" ]; do
-    EXISTS=$(cardano-cli latest query tx-mempool tx-exists "$TXID" | jq -re .exists)
+    EXISTS=$(cardano-cli latest query tx-mempool tx-exists "$TXID" | jq -re .exists || true)
     if [ "$EXISTS" = "true" ]; then
       echo "The transaction still exists in the mempool, sleeping 5s: $TXID"
     else
@@ -199,3 +199,30 @@ return-utxo() (
 
   submit "$BASENAME.signed"
 )
+
+# A handy faucet submission function with mempool monitoring, usable on custom networks.
+# CARDANO_NODE_{NETWORK_ID,SOCKET_PATH}, TESTNET_MAGIC should already be exported.
+# SEND_ADDR, LOVELACE, RICH_ADDR and RICH vars need to be provided.
+faucet() (
+  SEND_ADDR="$1"
+  LOVELACE="$2"
+
+  UTXOS=$(cardano-cli query utxo --address "$RICH_ADDR")
+  UTXO=$(jq -r 'to_entries | max_by(.value.value.lovelace) | { (.key): .value }' <<< "$UTXOS")
+  UTXO_TX=$(jq -r 'keys[0]' <<< "$UTXO")
+
+  cardano-cli latest transaction build \
+    --tx-in "$UTXO_TX" \
+    --tx-out "$SEND_ADDR+$LOVELACE" \
+    --change-address "$RICH_ADDR" \
+    --testnet-magic "$TESTNET_MAGIC" \
+    --out-file faucet.txbody
+
+  cardano-cli latest transaction sign \
+    --tx-body-file faucet.txbody \
+    --signing-key-file "$RICH.skey" \
+    --testnet-magic "$TESTNET_MAGIC" \
+    --out-file faucet.txsigned
+
+  submit faucet.txsigned
+)

templates/cardano-parts-project/scripts/lib/cli.py

@@ -53,7 +53,7 @@ def createTransaction(start, end, txin, payments_txouts, utxo_address, utxo_sign
 
     signed_tx = signTx(tx_prefix, utxo_signing_key_str)
 
-    p = subprocess.run([cardanoCliStr(), "latest", "transaction", "txid", "--tx-file", signed_tx], capture_output=True, text=True)
+    p = subprocess.run([cardanoCliStr(), "latest", "transaction", "txid", "--output-text", "--tx-file", signed_tx], capture_output=True, text=True)
     new_txin = p.stdout.rstrip()
     return (f"{new_txin}#0", tx_out_amount, fee)
 
@@ -66,6 +66,7 @@ def estimateFeeTx(txbody, txin_count, txout_count, pparams) -> int:
         "--tx-out-count", str(txout_count),
         "--witness-count", "1",
         "--protocol-params-file", pparams,
+        "--output-text",
         "--tx-body-file", txbody]
 
     p = subprocess.run(cmd, capture_output=True, text=True)

templates/cardano-parts-project/scripts/recipes/governance.just

@@ -992,7 +992,7 @@ vote-with-pool ENV POOL ACTION_ID ACTION_IDX VOTE:
   EXISTS="true"
 
   while [ "$EXISTS" = "true" ]; do
-    EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists)
+    EXISTS=$(eval "$CARDANO_CLI" latest query tx-mempool tx-exists $TXID | jq -r .exists || true)
     if [ "$EXISTS" = "true" ]; then
       echo "Vote transaction still exists in the mempool, sleeping 5s: $TXID"
     else

templates/cardano-parts-project/scripts/restore-delegation-accounts.py

@@ -261,7 +261,7 @@ def signTx(tx_body, utxo_signing_key_str, stake_signing_key, out_file):
     if p.returncode != 0:
         print(p.stderr)
         raise Exception("Unknown error signing transaction")
-    cli_args = ["cardano-cli", "latest", "transaction", "txid", "--tx-file", out_file]
+    cli_args = ["cardano-cli", "latest", "transaction", "txid", "--output-text", "--tx-file", out_file]
     p = subprocess.run(cli_args, input=None, capture_output=True, text=True)
     if p.returncode != 0:
         print(p.stderr)