Use RL9 for caas environment (#380)

* simplify caas zenith podman configuration

* use containers.conf to configure cgroup-manager for all containers (incl. zenith ones)

* add persist_hostkeys role

* fix persisted hostkeys not being authorised for OOD shell

* add missing persist_hostkeys README

Author: sjpb

ansible/extras.yml

@@ -26,3 +26,13 @@
   tasks:
     - import_role:
         name: cuda
+
+- name: Persist hostkeys across rebuilds
+  # Must be after filesystems.yml (for storage)
+  # and before portal.yml (where OOD login node hostkeys are scanned)
+  hosts: persist_hostkeys:!builder
+  become: yes
+  gather_facts: no
+  tasks:
+    - import_role:
+        name: persist_hostkeys

ansible/roles/filebeat/templates/filebeat.service.j2

@@ -12,7 +12,7 @@ After=network-online.target
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
-ExecStart=/usr/bin/podman --cgroup-manager=cgroupfs run \
+ExecStart=/usr/bin/podman run \
     --network=host \
     --sdnotify=conmon \
     --cgroups=no-conmon \

ansible/roles/mysql/templates/mysql.service.j2

@@ -14,7 +14,7 @@ EnvironmentFile=/etc/sysconfig/mysqld
 # The above EnvironmentFile must define MYSQL_INITIAL_ROOT_PASSWORD
 ExecStartPre=+install -d -o {{ mysql_podman_user }} -g {{ mysql_podman_user }} -Z container_file_t {{ mysql_datadir }}
 ExecStartPre=+chown -R {{ mysql_podman_user }}:{{ mysql_podman_user }} {{ mysql_datadir }}
-ExecStart=/usr/bin/podman --cgroup-manager=cgroupfs run \
+ExecStart=/usr/bin/podman run \
     --network=host \
     --sdnotify=conmon \
     --cgroups=no-conmon \

ansible/roles/opensearch/templates/opensearch.service.j2

@@ -11,7 +11,7 @@ Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 # paths below based on https://opensearch.org/docs/latest/opensearch/configuration/ and https://opensearch.org/docs/latest/security-plugin/configuration/yaml
 # see also https://opensearch.org/docs/2.0/opensearch/install/important-settings/
-ExecStart=/usr/bin/podman --cgroup-manager=cgroupfs run \
+ExecStart=/usr/bin/podman run \
     --network=host \
     --sdnotify=conmon \
     --cgroups=no-conmon \

ansible/roles/persist_hostkeys/README.md

@@ -0,0 +1,8 @@
+# persist_hostkeys
+
+Save hostkeys to persistent storage and restore them after a rebuild/reimage.
+
+Add hosts to the `persist_hostkeys` group to enable.
+
+This role has no variables but hosts in this group must have `appliances_state_dir`
+defined as a directory they can write to on persistent storage.

ansible/roles/podman/tasks/config.yml

@@ -26,6 +26,16 @@
     value: 25000000 # set same as root. Non-root default is 20000
   become: true
 
+- name: Configure podman to use cgroupfs as the cgroup manager
+  community.general.ini_file:
+  # is actually toml but there's no module for that
+    create: false # something's unexpected if it doesn't exist now
+    path: /usr/share/containers/containers.conf
+    section: engine
+    option: cgroup_manager
+    value: '"cgroupfs"'
+  become: true
+
 - name: reset ssh connection to allow user changes to affect 'current login user'
   meta: reset_connection
 

environments/.caas/hooks/post.yml

@@ -1,22 +1,5 @@
-- name: Persist login hostkey across rebuilds
-# Need NFS for this so can't do it before the appliance plays
-  hosts: login
-  gather_facts: no
-  become: yes
-  roles:
-    - persist_hostkeys
-
 # Configure the Zenith clients that are required
-# First, ensure that podman is installed on all hosts that will run Zenith clients
-- hosts: zenith,!podman
-  tasks:
-    - import_role:
-        name: podman
-        tasks_from: prereqs.yml
-    - import_role:
-        name: podman
-        tasks_from: config.yml
-
+# Note zenith hosts are in podman group
 - hosts: grafana
   tasks:
     - name: Deploy the Zenith client for Grafana

environments/.caas/inventory/extra_groups

@@ -11,3 +11,9 @@ openondemand
 [manila:children]
 login
 compute
+
+[podman:children]
+zenith
+
+[persist_hostkeys:children]
+openondemand

environments/common/inventory/groups

@@ -123,3 +123,6 @@ freeipa_client
 
 [proxy]
 # Hosts to configure http/s proxies - see ansible/roles/proxy/README.md
+
+[persist_hostkeys]
+# Hosts to persist hostkeys for across reimaging. NB: Requires appliances_state_dir on hosts.

environments/common/layouts/everything

@@ -69,3 +69,6 @@ openhpc
 
 [manila]
 # Hosts to configure for manila fileshares
+
+[persist_hostkeys]
+# Hosts to persist hostkeys for across reimaging. NB: Requires appliances_state_dir on hosts.