Support manila fileshares (cephfs) (#344)

sjpb · web-flow · commit f3f60b9e32d8 · 2024-01-31T09:10:43.000Z
* add support for manila to common environment * use manila share for /scratch in stackhpc env w/ tests * make home volume optional in skeleton TF * remove share creation from CI TF * add manila UI for caas * support manila- or nfs/volume- based home dirs in caas * remove manila config from UI * add optional platform-lifecycle manila share for homedirs for caas * add home and project manila config for caas * tweak home volume size UI description to account for shares * fix caas manila config typo * tidy PR diff * bump fatimage to include manila client * Revert commit "tweak home volume size UI description to account for shares" This reverts commit 3d9cfba. * add manila UI for caas * add defaults for new caas manila extravars, where possible * make cluster_home_manila_share_type optional for when default share type is defined * address review comments * bump manila requirement after role release * default usage of home manila share to match project share for caas * remove caas manila-specific ui-meta
diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml
@@ -41,10 +41,16 @@
         tasks_from: client-install.yml
       when: "'freeipa_client' in group_names"
 
-    # - import_playbook: filesystems.yml
-    - name: nfs
+    # - import_playbook: filesystems.yml:
+    - name: Install nfs packages
       dnf:
         name: nfs-utils
+      when: "'nfs' in group_names"
+    - name: Install Manila client packages
+      include_role:
+        name: stackhpc.os-manila-mount
+        tasks_from: install.yml
+      when: "'manila' in group_names"
 
 - import_playbook: extras.yml
 
diff --git a/ansible/filesystems.yml b/ansible/filesystems.yml
@@ -16,3 +16,11 @@
   tasks:
     - include_role:
         name: stackhpc.nfs
+
+- name: Setup Manila share mounts
+  hosts: manila
+  become: true
+  tags: manila
+  tasks:
+    - include_role:
+        name: stackhpc.os-manila-mount
diff --git a/ansible/roles/cluster_infra/templates/resources.tf.j2 b/ansible/roles/cluster_infra/templates/resources.tf.j2
@@ -79,6 +79,22 @@ resource "openstack_blockstorage_volume_v3" "state" {
     size = "{{ state_volume_size }}"
 }
 
+{% if cluster_home_manila_share | bool %}
+resource "openstack_sharedfilesystem_share_v2" "home" {
+  name = "{{ cluster_name }}-home"
+  description = "Home for cluster"
+  share_proto = "CEPHFS"
+  share_type = {{ '"' + cluster_home_manila_share_type + '"' | default('null') }}
+  size = "{{ home_volume_size }}"
+}
+
+resource "openstack_sharedfilesystem_share_access_v2" "home" {
+  share_id     = openstack_sharedfilesystem_share_v2.home.id
+  access_type  = "cephx"
+  access_to    = "cluster_{{ cluster_id }}"
+  access_level = "rw"
+}
+{% else %}
 resource "openstack_blockstorage_volume_v3" "home" {
     name = "{{ cluster_name }}-home"
     description = "Home for control node"
@@ -89,6 +105,7 @@ resource "openstack_blockstorage_volume_v3" "home" {
     {% endif %}
     {% endif %}
 }
+{% endif %}
 
 ######
 ###### Cluster network
@@ -334,13 +351,15 @@ resource "openstack_compute_instance_v2" "control" {
       uuid = openstack_blockstorage_volume_v3.state.id
   }
 
+  {% if not cluster_home_manila_share | bool %}
   # home volume:
   block_device {
       destination_type = "volume"
       source_type  = "volume"
       boot_index = -1
       uuid = openstack_blockstorage_volume_v3.home.id
   }
+  {% endif %}
 
   # Use cloud-init to a) inject SSH keys b) configure volumes
   user_data = <<-EOF
@@ -359,12 +378,14 @@ resource "openstack_compute_instance_v2" "control" {
       - {{ ssh_key }}
     {%- endfor %}
     bootcmd:
-    %{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
+    %{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
     - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
     %{endfor}
     mounts:
         - [LABEL=state, {{ appliances_state_dir }}, auto]
+        {% if not cluster_home_manila_share | bool %}
         - [LABEL=home, /exports/home, auto]
+        {% endif %}
   EOF
 }
 
diff --git a/environments/.caas/inventory/extra_groups b/environments/.caas/inventory/extra_groups
@@ -7,3 +7,7 @@ cluster
 [zenith:children]
 grafana
 openondemand
+
+[manila:children]
+login
+compute
diff --git a/environments/.caas/inventory/group_vars/all/cluster.yml b/environments/.caas/inventory/group_vars/all/cluster.yml
@@ -20,3 +20,7 @@ openondemand_servername_default: "{{ hostvars[groups['openstack'][0]].cluster_ga
 openondemand_servername: "{{ zenith_fqdn_ood | default(openondemand_servername_default) }}"
 
 appliances_state_dir: /var/lib/state
+
+# Defaults for caas-provided extravars:
+cluster_project_manila_share: false
+cluster_home_manila_share: "{{ cluster_project_manila_share }}"
diff --git a/environments/.caas/inventory/group_vars/all/manila.yml b/environments/.caas/inventory/group_vars/all/manila.yml
@@ -0,0 +1,16 @@
+caas_manila_home:
+  share_name: "{{ cluster_name }}-home"
+  mount_path: /home
+  mount_user: root
+  mount_group: root
+  mount_mode: u=rwX,go=rX
+
+cluster_project_manila_share_name: azimuth-project-share
+caas_manila_project:
+  share_name: "{{ cluster_project_manila_share_name }}"
+  mount_path: /project
+  mount_user: root
+  mount_group: root
+  mount_mode: ugo=rwX
+
+os_manila_mount_shares: "{{ ([caas_manila_home] if cluster_home_manila_share | bool else []) + ([caas_manila_project] if cluster_project_manila_share | bool else []) }}"
diff --git a/environments/.caas/inventory/group_vars/all/nfs.yml b/environments/.caas/inventory/group_vars/all/nfs.yml
@@ -1,16 +1,20 @@
 nfs_server: "{{ nfs_server_default }}"
 
-nfs_configurations:
-  - comment: Export /exports/home from Slurm control node as /home
-    nfs_enable:
-        server:  "{{ inventory_hostname in groups['control'] }}"
-        clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
-    nfs_export: "/exports/home" # assumes skeleton TF is being used
-    nfs_client_mnt_point: "/home"
+caas_nfs_ood_state:
   - comment: Export /var/lib/state from Slurm control node to OOD
     nfs_enable:
         server:  "{{ inventory_hostname in groups['control'] }}"
         clients: "{{ inventory_hostname in groups['openondemand'] }}"
     nfs_export: "{{ appliances_state_dir }}"
     nfs_client_mnt_point: "{{ appliances_state_dir }}"
     nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"
+
+caas_nfs_home:
+  - comment: Export /exports/home from Slurm control node as /home
+    nfs_enable:
+        server:  "{{ inventory_hostname in groups['control'] }}"
+        clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
+    nfs_export: "/exports/home" # assumes skeleton TF is being used
+    nfs_client_mnt_point: "/home"
+
+nfs_configurations: "{{ caas_nfs_ood_state + (caas_nfs_home if not cluster_home_manila_share | bool else []) }}"
diff --git a/environments/.caas/ui-meta/slurm-infra.yml b/environments/.caas/ui-meta/slurm-infra.yml
@@ -50,7 +50,7 @@ parameters:
   
   - name: home_volume_size
     label: Home volume size (GB)
-    description: The size of the cloud volume to use for home directories.
+    description: The size of the cloud volume or share to use for home directories.
     kind: cloud.volume_size
     immutable: true
     options:
diff --git a/environments/.stackhpc/inventory/extra_groups b/environments/.stackhpc/inventory/extra_groups
@@ -22,3 +22,8 @@ cluster
 # [resolv_conf:children]
 # freeipa_client
 # --- end of FreeIPA example ---
+
+[manila:children]
+# Allows demo; also installs manila client in fat image
+login
+compute
diff --git a/environments/.stackhpc/terraform/main.tf b/environments/.stackhpc/terraform/main.tf
@@ -1,5 +1,14 @@
 # This terraform configuration uses the "skeleton" terraform, so that is checked by CI.
 
+terraform {
+  required_version = ">= 0.14"
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}
+
 variable "environment_root" {
     type = string
     description = "Path to environment root, automatically set by activate script"
@@ -13,7 +22,7 @@ variable "cluster_name" {
 variable "cluster_image" {
     description = "single image for all cluster nodes - a convenience for CI"
     type = string
-    default = "openhpc-240116-1156-aa8dba7d" # https://github.com/stackhpc/ansible-slurm-appliance/pull/351
+    default = "openhpc-240116-1604-b3563a08" # https://github.com/stackhpc/ansible-slurm-appliance/pull/344
     # default = "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2"
 }
 
diff --git a/environments/common/inventory/group_vars/all/manila.yml b/environments/common/inventory/group_vars/all/manila.yml
@@ -0,0 +1,13 @@
+# Default configuration for manila file shares, see
+# https://github.com/stackhpc/ansible-role-os-manila-mount
+# for all variable definitions, and override in your environment.
+
+os_manila_mount_shares: []
+  # - share_name:
+  #   share_user:
+  #   mount_path:
+  #   mount_user:
+  #   mount_group:
+  #   mount_mode:
+
+# os_manila_mount_ceph_version: nautilus # role default for RockyLinux 8
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -66,3 +66,6 @@ openhpc
 
 [proxy]
 # Hosts to configure http/s proxies - see ansible/roles/proxy/README.md
+
+[manila]
+# Hosts to configure for manila fileshares
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf
@@ -1,5 +1,6 @@
 locals {
   user_data_path = "${var.environment_root}/cloud_init/${var.cluster_name}-%s.userdata.yml"
+  control_volumes = concat([openstack_blockstorage_volume_v3.state], var.home_volume_size > 0 ? [openstack_blockstorage_volume_v3.home][0] : [])
 }
 
 
@@ -100,20 +101,14 @@ resource "openstack_compute_instance_v2" "control" {
       delete_on_termination = true
   }
 
-  # state volume:
-  block_device {
-      destination_type = "volume"
-      source_type  = "volume"
-      boot_index = -1
-      uuid = openstack_blockstorage_volume_v3.state.id
-  }
-
-  # home volume:
-  block_device {
+  dynamic "block_device" {
+    for_each = local.control_volumes
+    content {
       destination_type = "volume"
       source_type  = "volume"
       boot_index = -1
-      uuid = openstack_blockstorage_volume_v3.home.id
+      uuid = block_device.value.id # actually openstack_blockstorage_volume_v3 id
+    }
   }
 
   network {
@@ -130,13 +125,15 @@ resource "openstack_compute_instance_v2" "control" {
     fqdn: ${var.cluster_name}-${each.key}.${var.cluster_name}.${var.cluster_domain_suffix}
     
     bootcmd:
-      %{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
+      %{for volume in local.control_volumes}
       - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
       %{endfor}
 
     mounts:
       - [LABEL=state, ${var.state_dir}]
+      %{if var.home_volume_size > 0}
       - [LABEL=home, /exports/home]
+      %{endif}
   EOF
 
 }
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf
@@ -70,7 +70,7 @@ variable "state_volume_size" {
 variable "home_volume_size" {
     type = number
     description = "Size of state volume on control node, in GB"
-    default = 100 # GB
+    default = 100 # GB, 0 means no home volume
 }
 
 variable "vnic_type" {
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/volumes.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/volumes.tf
@@ -5,6 +5,9 @@ resource "openstack_blockstorage_volume_v3" "state" {
 }
 
 resource "openstack_blockstorage_volume_v3" "home" {
+
+    count = var.home_volume_size > 0 ? 1 : 0
+
     name = "${var.cluster_name}-home"
     description = "Home for control node" # first word used to label filesystem
     size = var.home_volume_size
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,7 @@
 ansible==6.0.0
 openstacksdk
 python-openstackclient
+python-manilaclient
 jmespath
 passlib[bcrypt]==1.7.4
 cookiecutter
diff --git a/requirements.yml b/requirements.yml
@@ -20,6 +20,9 @@ roles:
   - src: https://github.com/OSC/ood-ansible.git
     name: osc.ood
     version: v3.0.6
+  - src: https://github.com/stackhpc/ansible-role-os-manila-mount.git
+    name: stackhpc.os-manila-mount
+    version: v24.1.0
 
 collections:
   - name: containers.podman
