Enable CI on leafcloud (#369)

* improve setup script failures

* provide default for vnic type

* add infra configuration for leafcloud

* wip: debug workflow

* allow PR labels of CI_CLOUD= to override repo-level CI_CLOUD

* add leafcloud settings

* remove workflow debug

* enable setting volume types

* remove SMS CI info

* switch CI to single partition

* downsize Arcus CI control node

* Revert "allow PR labels of CI_CLOUD= to override repo-level CI_CLOUD"

This reverts commit c0c3575.

* allow using CI_CLOUD=LEAFCLOUD in PR labels to override CI_CLOUD

* revert "allow using CI_CLOUD=LEAFCLOUD in PR labels to override CI_CLOUD"

This reverts commit 89ee4b1.

* use opentofu for CI

* fix expected sinfo output in CI

* fix volume_type defaults

---------

Co-authored-by: Cloud User <rocky@steveb-dev.novalocal>

Author: sjpb

.github/workflows/stackhpc.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Record which cloud CI is running on
+      - name: Record settings for CI cloud
         run: |
           echo CI_CLOUD: ${{ vars.CI_CLOUD }}
 
@@ -38,10 +38,10 @@ jobs:
       - name: Install ansible etc
         run: dev/setup-env.sh
 
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v1
+      - name: Install OpenTofu
+        uses: opentofu/setup-opentofu@v1
         with:
-          terraform: v1.5.5
+          tofu_version: 1.6.2
 
       - name: Initialise terraform
         run: terraform init

ansible/ci/check_slurm.yml

@@ -21,5 +21,4 @@
           <end>
       vars:
         expected_sinfo:
-          - "{{ openhpc_cluster_name }}-compute-[0-1] small* up 60-00:00:00 2 idle"
-          - "{{ openhpc_cluster_name }}-compute-[2-3] extra up 60-00:00:00 2 idle"
+          - "{{ openhpc_cluster_name }}-compute-[0-1] standard* up 60-00:00:00 2 idle"

dev/setup-env.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euo pipefail
+
 if [[ ! -d "venv" ]]; then
     /usr/bin/python3.8 -m venv venv # use `sudo yum install python38` on Rocky Linux 8 to install this
 fi

environments/.stackhpc/SMS.pkrvars.hcl

@@ -2,3 +2,4 @@
 |1|whGSPLhKW4xt/7PWOZ1treg3PtA=|F5gwV8j0JYWDzjb6DvHHaqO+sxs= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpCG881Gt3dr+nuVIC2uGEQkeVwG6WDdS1WcCoxXC7AG+Oi5bfdqtf4IfeLpWmeuEaAaSFH48ODFr76ViygSjU=
 |1|0V6eQ1FKO5NMKaHZeNFbw62mrJs=|H1vuGTbbtZD2MEgZxQf1PXPk+yU= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEnOtYByM3s2qvRT8SS1sn5z5sbwjzb1alm0B3emPcHJ
 |1|u3QVAK9R2x7Z3uKNj+0vDEIekl0=|yy09Q0Kw472+J7bjFkmir28x3lE= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINNuXZkH7ppkTGNGKzmGEvAnvlLO2D+YtlJw1m3P16FV
+|1|nOHeibGxhsIFnhW0flRwnirJjlg=|IJ8nJB355LGI+1U3Wpvdcgdf0ek= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGG6DieKAdgiTCqRmF2HD0dJi9DuORblPzbridniICsw

environments/.stackhpc/bastion_fingerprints

@@ -3,9 +3,9 @@ bastion_config:
   ARCUS:
     user: slurm-app-ci
     ip: 128.232.222.183
-  SMS:
-    user: steveb
-    ip: 185.45.78.150
+  LEAFCLOUD:
+    user: rocky
+    ip: 195.114.30.222
 # NB: The bastion_{user,ip} variables are used directly in the CI workflow too
 bastion_user: "{{ bastion_config[ci_cloud].user }}"
 bastion_ip: "{{ bastion_config[ci_cloud].ip }}"

environments/.stackhpc/inventory/group_vars/all/bastion.yml

@@ -1,5 +1,4 @@
 cluster_net = "portal-internal"
 cluster_subnet = "portal-internal"
-vnic_type = "normal"
-control_node_flavor = "vm.ska.cpu.general.quarter"
+control_node_flavor = "vm.ska.cpu.general.eighth"
 other_node_flavor = "vm.ska.cpu.general.small"

environments/.stackhpc/terraform/ARCUS.tfvars

@@ -0,0 +1,6 @@
+cluster_net = "slurmapp-ci"
+cluster_subnet = "slurmapp-ci"
+control_node_flavor = "ec1.medium" # small ran out of memory, medium gets down to ~100Mi mem free on deployment
+other_node_flavor = "ec1.small"
+state_volume_type = "unencrypted"
+home_volume_type = "unencrypted"

environments/.stackhpc/terraform/LEAFCLOUD.tfvars

@@ -30,7 +30,17 @@ variable "cluster_net" {}
 
 variable "cluster_subnet" {}
 
-variable "vnic_type" {}
+variable "vnic_type" {
+    default = "normal"
+}
+
+variable "state_volume_type"{
+    default = null
+}
+
+variable "home_volume_type"{
+    default = null
+}
 
 variable "control_node_flavor" {}
 
@@ -59,20 +69,21 @@ module "cluster" {
         }
     }
     compute_types = {
-        small: {
-            flavor: var.other_node_flavor
-            image: var.cluster_image
-        }
-        extra: {
+        standard: { # NB: can't call this default!
             flavor: var.other_node_flavor
             image: var.cluster_image
         }
+        # Example of how to add another partition:
+        # extra: {
+        #     flavor: var.other_node_flavor
+        #     image: var.cluster_image
+        # }
     }
     compute_nodes = {
-        compute-0: "small"
-        compute-1: "small"
-        compute-2: "extra"
-        compute-3: "extra"
+        compute-0: "standard"
+        compute-1: "standard"
+        # compute-2: "extra"
+        # compute-3: "extra"
     }
     volume_backed_instances = var.volume_backed_instances
 
@@ -81,4 +92,7 @@ module "cluster" {
     state_volume_size = 10
     home_volume_size = 20
 
+    state_volume_type = var.state_volume_type
+    home_volume_type = var.home_volume_type
+
 }