Merge pull request #143 from stackhpc/fix/padded-nodenames

Fix slurm.conf inventory filtering for nodenames with padding

Author: sjpb

filter_plugins/slurm_conf.py

@@ -38,17 +38,19 @@ def hostlist_expression(hosts):
         E.g. with an inventory containing:
 
             [compute]
-            dev-foo-0 ansible_host=localhost
-            dev-foo-3 ansible_host=localhost
+            dev-foo-00 ansible_host=localhost
+            dev-foo-3  ansible_host=localhost
             my-random-host
-            dev-foo-4 ansible_host=localhost
-            dev-foo-5 ansible_host=localhost
-            dev-compute-0 ansible_host=localhost
-            dev-compute-1 ansible_host=localhost
+            dev-foo-04 ansible_host=localhost
+            dev-foo-05 ansible_host=localhost
+            dev-compute-000 ansible_host=localhost
+            dev-compute-001 ansible_host=localhost
 
         Then "{{ groups[compute] | hostlist_expression }}" will return:
             
-            ["dev-foo-[0,3-5]", "dev-compute-[0-1]", "my-random-host"]
+            ['dev-foo-[00,04-05,3]', 'dev-compute-[000-001]', 'my-random-host']
+
+        NB: This does not guranteed to return parts in the same order as `scontrol hostlist`, but its output should return the same hosts when passed to `scontrol hostnames`.
     """
 
     results = {}
@@ -58,19 +60,23 @@ def hostlist_expression(hosts):
         if m:
             prefix, suffix = m.groups()
             r = results.setdefault(prefix, [])
-            r.append(int(suffix))
+            r.append(suffix)
         else:
             unmatchable.append(v)
     return ['{}[{}]'.format(k, _group_numbers(v)) for k, v in results.items()] + unmatchable
 
 def _group_numbers(numbers):
     units = []
-    prev = min(numbers)
-    for v in sorted(numbers):
+    ints = [int(n) for n in numbers]
+    lengths = [len(n) for n in numbers]
+    # sort numbers by int value and length:
+    ints, lengths, numbers = zip(*sorted(zip(ints, lengths, numbers)))
+    prev = min(ints)
+    for i, v in enumerate(sorted(ints)):
         if v == prev + 1:
-            units[-1].append(v)
+            units[-1].append(numbers[i])
         else:
-            units.append([v])
+            units.append([numbers[i]])
         prev = v
     return ','.join(['{}-{}'.format(u[0], u[-1]) if len(u) > 1 else str(u[0]) for u in units])
 

molecule/README.md

@@ -26,7 +26,7 @@ test15 | 1            | N                       | No compute nodes.
 
 # Local Installation & Running
 
-Local installation on a CentOS 8 machine looks like:
+Local installation on a RockyLinux 8.x machine looks like:
 
     sudo yum install -y gcc python3-pip python3-devel openssl-devel python3-libselinux
     sudo yum install -y yum-utils
@@ -35,8 +35,7 @@ Local installation on a CentOS 8 machine looks like:
     sudo yum install -y iptables
     sudo systemctl start docker
     sudo usermod -aG docker ${USER}
-    # if not running as centos, also run:
-    sudo usermod -aG docker centos
+    sudo usermod -aG docker rocky
     newgrp docker
     docker run hello-world # test docker works without sudo
 

molecule/test12/molecule.yml

@@ -16,6 +16,10 @@ platforms:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro
     networks:
       - name: net1
+    docker_networks:
+      - name: net1
+        driver_options:
+          com.docker.network.driver.mtu: ${DOCKER_MTU:-1500} # 1500 is docker default
   - name: testohpc-compute-0
     image: ${MOLECULE_IMAGE}
     pre_build_image: true

molecule/test12/verify.yml

@@ -23,6 +23,9 @@
       cmd: "sacct --completion --noheader --parsable2"
     changed_when: false
     register: sacct
+    until: "sacct.stdout.strip() != ''"
+    retries: 5
+    delay: 1
   - assert:
       that: "(jobid + '|0|wrap|compute|2|testohpc-compute-[0-1]|COMPLETED') in sacct.stdout"
       fail_msg: "Didn't find expected output for {{ jobid }} in sacct output: {{ sacct.stdout }}"

tests/filter.yml

@@ -5,15 +5,19 @@
   vars:
     grouped_0: "{{ groups['mock_group_0'] | hostlist_expression }}"
     grouped_1: "{{ groups['mock_group_1'] | hostlist_expression }}"
+    grouped_2: "{{ groups['mock_group_2'] | hostlist_expression }}"
   tasks:
-    - name: Show grouped mock-group-0
-      debug: var=grouped_0
-    - name: Show grouped mock-group-1
-      debug: var=grouped_1
     - name: Test filter
       assert:
-        that:
-          - "['localhost-0-[0-3,5]', 'localhost-non-numerical'] == grouped_0"
-          - "['localhost-1-[1-2,4-5,10]', 'localhost-2-[1-3]'] == grouped_1"
-
+        that: item.result == item.expected
+        fail_msg: |
+          expected: {{ item.expected }}
+          got: {{ item.result }}
+      loop:
+        - result: "{{ grouped_0 }}"
+          expected: ['localhost-0-[0-3,5]', 'localhost-non-numerical']
+        - result: "{{ grouped_1 }}"
+          expected: ['localhost-1-[1-2,4-5,10]', 'localhost-2-[1-3]']
+        - result: "{{ grouped_2 }}"
+          expected: ['localhost-[1,0001-0003,0008,0010]', 'localhost-admin']
 ...

tests/inventory-mock-groups

@@ -15,3 +15,15 @@ localhost-1-10 ansible_host=127.0.0.1  ansible_connection='local'    ansible_pyt
 localhost-2-1 ansible_host=127.0.0.1  ansible_connection='local'    ansible_python_interpreter='/usr/bin/env python'
 localhost-2-2 ansible_host=127.0.0.1  ansible_connection='local'    ansible_python_interpreter='/usr/bin/env python'
 localhost-2-3 ansible_host=127.0.0.1  ansible_connection='local'    ansible_python_interpreter='/usr/bin/env python'
+
+[mock_group_2]
+# test padding:
+# $ scontrol show hostlist localhost-admin,localhost-0001,localhost-0002,localhost-0003,localhost-0008,localhost-0010,localhost-1
+# localhost-admin,localhost-[0001-0003,0008,0010,1]
+localhost-0001
+localhost-1
+localhost-0010
+localhost-admin
+localhost-0002
+localhost-0008
+localhost-0003