Merge pull request #316 from stackhpc/feat/dev-qol

Development quality-of-life improvements

Author: sjpb

.github/workflows/fatimage.yml

@@ -47,7 +47,7 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=ask -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
 
       - name: Get created image name from manifest
         id: manifest

ansible/ci/retrieve_inventory.yml

@@ -7,7 +7,8 @@
   gather_facts: no
   vars:
     cluster_prefix: "{{ undef(hint='cluster_prefix must be defined') }}" # e.g. ci4005969475
-    cluster_network: WCDC-iLab-60
+    ci_vars_file: "{{ appliances_environment_root + '/terraform/' + lookup('env', 'CI_CLOUD') }}.tfvars"
+    cluster_network: "{{ lookup('ansible.builtin.ini', 'cluster_net', file=ci_vars_file, type='properties') | trim('\"') }}"
   tasks:
     - name: Get control host IP
       set_fact:

ansible/roles/mysql/tasks/configure.yml

@@ -21,7 +21,7 @@
     community.mysql.mysql_info:
       login_user: root
       login_password: "{{ mysql_root_password }}"
-    # no_log: true # TODO: FIXME
+    no_log: "{{ no_log | default(true) }}"
     register: _mysql_info
     until: "'version' in _mysql_info"
     retries: 90

dev/delete-cluster.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+"""
+Delete infrastructure for a cluster without using Terraform. Useful for CI clusters.
+
+Usage:
+    delete-cluster.py PREFIX
+
+Where PREFIX is the string at the start of the resource's names.
+It will list matching resources and prompt to confirm deletion.
+"""
+
+import sys, json, subprocess, pprint
+
+
+CLUSTER_RESOURCES = ['server', 'port', 'volume']
+
+def delete_cluster(cluster_prefix):
+    to_delete = {}
+    for resource_type in CLUSTER_RESOURCES:
+        to_delete[resource_type] = []
+        resource_list = subprocess.run(f'openstack {resource_type} list --format json', stdout=subprocess.PIPE, shell=True)
+        resources = json.loads(resource_list.stdout)
+        for item in resources:
+            try:
+                if item['Name'] is not None and item['Name'].startswith(cluster_prefix):
+                    print(resource_type, item['Name'], item['ID'])
+                    to_delete[resource_type].append(item)
+            except:
+                print(resource_type, item)
+                raise
+    if input('Delete these (y/n)?:') == 'y':
+        for resource_type in CLUSTER_RESOURCES:
+            items = [v['ID'] for v in to_delete[resource_type]]
+            if items:
+                # delete all resources of each type in a single call for speed:
+                subprocess.run(f"openstack {resource_type} delete {' '.join(items)}", stdout=subprocess.PIPE, shell=True)
+                print(f'Deleted {len(items)} {resource_type}s')
+    else:
+        print('Cancelled - no resources deleted')
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print('ERROR: Incorrect argument(s).\n' + __doc__)
+        exit(1)
+    delete_cluster(sys.argv[1])

dev/setup-env.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 
-/usr/bin/python3.8 -m venv venv # use `sudo yum install python38` on Rocky Linux 8 to install this
+if [[ ! -d "venv" ]]; then
+    /usr/bin/python3.8 -m venv venv # use `sudo yum install python38` on Rocky Linux 8 to install this
+fi
 . venv/bin/activate
 pip install -U pip
 pip install -r requirements.txt
 ansible --version
-# Install ansible dependencies ...
-ansible-galaxy role install -r requirements.yml -p ansible/roles
-ansible-galaxy collection install -r requirements.yml -p ansible/collections
+# Install or update ansible dependencies ...
+ansible-galaxy role install -fr requirements.yml -p ansible/roles
+ansible-galaxy collection install -fr requirements.yml -p ansible/collections

environments/.stackhpc/ansible.cfg

@@ -2,6 +2,7 @@
 any_errors_fatal = True
 stdout_callback = debug
 stderr_callback = debug
+callbacks_enabled = ansible.posix.profile_tasks
 gathering = smart
 forks = 30
 host_key_checking = False

environments/.stackhpc/hooks/pre.yml

@@ -10,7 +10,7 @@
         mode: 0400
         owner: root
         group: root
-      no_log: true
+      no_log: "{{ no_log | default(true) }}"
       loop:
         - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/hosts"
         - "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/all/secrets.yml"