From 0ce0123bafcd48366f7c0f61aa69a13be1ecb908 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 17 Feb 2025 15:19:51 +0000
Subject: [PATCH 01/10] Automated Kolla dependencies updates

---
 .github/workflows/package-build-ofed.yml  | 279 ++++------------------
 .github/workflows/update-dependencies.yml |  92 +++++++
 2 files changed, 139 insertions(+), 232 deletions(-)
 create mode 100644 .github/workflows/update-dependencies.yml

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 798e0c4bf2..4aef3ddbce 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -1,46 +1,36 @@
----
-name: Build OFED packages
+name: Update dependencies
+
 on:
+  # Allow manual executions
   workflow_dispatch:
-    inputs:
-      rocky9:
-        description: Build Rocky Linux 9
-        type: boolean
-        default: true
-    secrets:
-      KAYOBE_VAULT_PASSWORD:
-        required: true
-      CLOUDS_YAML:
-        required: true
-      OS_APPLICATION_CREDENTIAL_ID:
-        required: true
-      OS_APPLICATION_CREDENTIAL_SECRET:
-        required: true
 
-env:
-  ANSIBLE_FORCE_COLOR: True
-  KAYOBE_ENVIRONMENT: ci-builder
-  KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
-  overcloud-ofed-packages:
-    name: Build OFED packages
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: arc-skc-host-image-builder-runner
-    permissions: {}
+  propose_github_release_updates:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - key: kolla
+            repo_path: ${{ github.workspace }}/src/kolla
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla
+            version_jsonpath: stackhpc_kolla_source_version
+
+          - key: kolla-ansible
+            repo_path: ${{ github.workspace }}/src/kolla-ansible
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla-ansible
+            version_jsonpath: stackhpc_kolla_ansible_source_version
+
+          - key: kayobe
+            repo_path: ${{ github.workspace }}/src/kayobe
+            path: ${{ github.workspace }}/src/kayobe-config/requirements.txt
+            repository: stackhpc/kayobe
+    name: ${{ matrix.key }}
     steps:
-      - name: Install Package
-        uses: ConorMacBride/install-package@main
-        with:
-          apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
-
-      - name: Start the SSH service
-        run: |
-          sudo /etc/init.d/ssh start
-
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          path: src/kayobe-config
+          path: ${{ github.workspace }}/src/kayobe-config
 
       - name: Determine OpenStack release
         id: openstack_release
@@ -48,207 +38,32 @@ jobs:
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
-      - name: Clone StackHPC Kayobe repository
+      - name: Checkout the dependency repo
         uses: actions/checkout@v4
         with:
-          repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
-          path: src/kayobe
-
-      - name: Install Kayobe
-        run: |
-          mkdir -p venvs &&
-          pushd venvs &&
-          python3 -m venv kayobe &&
-          source kayobe/bin/activate &&
-          pip install -U pip &&
-          pip install ../src/kayobe
-
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v2
-
-      - name: Initialise terraform
-        run: terraform init
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Generate SSH keypair
-        run: ssh-keygen -f id_rsa -N ''
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Generate clouds.yaml
-        run: |
-          cat << EOF > clouds.yaml
-          ${{ secrets.CLOUDS_YAML }}
-          EOF
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Output image tag
-        id: image_tag
-        run: |
-          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
-
-      # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
-      - name: Output image name
-        id: image_name
-        run: |
-          echo image_name=overcloud-rocky-9-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
-
-      - name: Generate terraform.tfvars
-        run: |
-          cat << EOF > terraform.tfvars
-          ssh_public_key = "id_rsa.pub"
-          ssh_username = "cloud-user"
-          aio_vm_name = "skc-ofed-builder"
-          aio_vm_image = "${{ env.VM_IMAGE }}"
-          aio_vm_flavor = "en1.medium"
-          aio_vm_network = "stackhpc-ci"
-          aio_vm_subnet = "stackhpc-ci"
-          aio_vm_interface = "ens3"
-          EOF
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-        env:
-          VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
-
-      - name: Terraform Plan
-        run: terraform plan
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-        env:
-          OS_CLOUD: "openstack"
-          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
-          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-
-      - name: Terraform Apply
-        run: |
-          for attempt in $(seq 5); do
-              if terraform apply -auto-approve; then
-                  echo "Created infrastructure on attempt $attempt"
-                  exit 0
-              fi
-              echo "Failed to create infrastructure on attempt $attempt"
-              sleep 10
-              terraform destroy -auto-approve
-              sleep 60
-          done
-          echo "Failed to create infrastructure after $attempt attempts"
-          exit 1
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-        env:
-          OS_CLOUD: "openstack"
-          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
-          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-
-      - name: Get Terraform outputs
-        id: tf_outputs
-        run: |
-          terraform output -json
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
-
-      - name: Write Terraform outputs
-        run: |
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
-          ${{ steps.tf_outputs.outputs.stdout }}
-          EOF
+          repository: ${{ matrix.repository }}
+          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          path: ${{ matrix.repo_path }}
 
-      - name: Write Terraform network config
+      - name: Get latest tag
+        id: latest_tag
         run: |
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
-          ---
-          aio_ips:
-            builder: "{{ access_ip_v4.value }}"
-          EOF
+          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
+          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
+        working-directory: ${{ matrix.repo_path }}
 
-      - name: Write Terraform network interface config
-        run: |
-          mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
-          rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
-          admin_interface: "{{ access_interface.value }}"
-          aio_interface: "{{ access_interface.value }}"
-          EOF
-
-      - name: Manage SSH keys
-        run: |
-          mkdir -p ~/.ssh
-          touch ~/.ssh/authorized_keys
-          cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
-          cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
-
-      - name: Bootstrap the control host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe control host bootstrap
-
-      - name: Run growroot playbook
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Configure the seed host (Builder VM)
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Run a distro-sync
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Reset BLS entries on the seed host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reset-bls-entries.yml \
-          -e "reset_bls_host=ofed-builder"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Disable noexec in /var/tmp
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "sed -i 's/noexec,//g' /etc/fstab"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Reboot to apply the kernel update
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reboot.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Run OFED builder playbook
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+      - name: Update dependency key (kolla)
+        uses: azimuth-cloud/github-actions/config-update@master
+        with:
+          path: ${{ matrix.path }}
+          updates: |
+            ${{ matrix.version_jsonpath }}=${{ steps.latest_tag.outputs.latest_tag }}
+        if: contains(matrix.key, 'kolla')
 
-      - name: Run OFED upload playbook
+      - name: Update dependency key (kayobe)
         run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-
-      - name: Destroy
-        run: terraform destroy -auto-approve
-        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+          REPLACE=$(sed -i "s/@stackhpc\/.*$/$TAG/g" $REQUIREMENTS)
+        if: contains(matrix.key, 'kayobe')
         env:
-          OS_CLOUD: openstack
-          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
-          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
-        if: always()
+          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
+          REQUIREMENTS: ${{ matrix.path }}
diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml
new file mode 100644
index 0000000000..8184792466
--- /dev/null
+++ b/.github/workflows/update-dependencies.yml
@@ -0,0 +1,92 @@
+name: Update dependencies
+
+on:
+  # Allow manual executions
+  workflow_dispatch:
+  # Run nightly
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  propose_github_release_updates:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - key: kolla
+            repo_path: ${{ github.workspace }}/src/kolla
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla
+            version_jsonpath: stackhpc_kolla_source_version
+
+          - key: kolla-ansible
+            repo_path: ${{ github.workspace }}/src/kolla-ansible
+            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla-ansible
+            version_jsonpath: stackhpc_kolla_ansible_source_version
+
+          - key: kayobe
+            repo_path: ${{ github.workspace }}/src/kayobe
+            path: ${{ github.workspace }}/src/kayobe-config/requirements.txt
+            repository: stackhpc/kayobe
+    permissions:
+      contents: write
+      pull-requests: write
+    name: ${{ matrix.key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+          path: ${{ github.workspace }}/src/kayobe-config
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+
+      - name: Checkout the dependency repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.repository }}
+          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          path: ${{ matrix.repo_path }}
+
+      - name: Get latest tag
+        id: latest_tag
+        run: |
+          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
+          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
+        working-directory: ${{ matrix.repo_path }}
+
+      - name: Update dependency key (kolla)
+        uses: azimuth-cloud/github-actions/config-update@master
+        with:
+          path: ${{ matrix.path }}
+          updates: |
+            ${{ matrix.version_jsonpath }}=${{ steps.latest_tag.outputs.latest_tag }}
+        if: contains(matrix.key, 'kolla')
+
+      - name: Update dependency key (kayobe)
+        run: |
+          REPLACE=$(sed -i "s/@stackhpc\/.*$/$TAG/g" $REQUIREMENTS)
+        if: contains(matrix.key, 'kayobe')
+        env:
+          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
+          REQUIREMENTS: ${{ matrix.path }}
+
+      - name: Propose changes via PR if required
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+          path: ${{ github.workspace }}/src/kayobe-config
+          commit-message: >-
+            Bump ${{ matrix.key }} to ${{ steps.next.outputs.version }}
+          branch: update-dependency/${{ matrix.key }}
+          delete-branch: true
+          title: >-
+            Bump ${{ matrix.key }} to ${{ steps.next.outputs.version }}
+          body: >
+            This PR was created automatically to update
+            ${{ matrix.key }} to ${{ steps.next.outputs.version }}.
+          labels: |
+            stackhpc-ci

From 8960c275bf191cb1522d3588ec6e6816da99e675 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 19 Feb 2025 10:06:14 +0000
Subject: [PATCH 02/10] Test syntax fix

---
 .github/workflows/package-build-ofed.yml  |  8 +-
 .github/workflows/update-dependencies.yml | 92 -----------------------
 2 files changed, 1 insertion(+), 99 deletions(-)
 delete mode 100644 .github/workflows/update-dependencies.yml

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 4aef3ddbce..197c0a9cec 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -32,17 +32,11 @@ jobs:
         uses: actions/checkout@v4
           path: ${{ github.workspace }}/src/kayobe-config
 
-      - name: Determine OpenStack release
-        id: openstack_release
-        run: |
-          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
-
       - name: Checkout the dependency repo
         uses: actions/checkout@v4
         with:
           repository: ${{ matrix.repository }}
-          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          ref: stackhpc/2024.1
           path: ${{ matrix.repo_path }}
 
       - name: Get latest tag
diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml
deleted file mode 100644
index 8184792466..0000000000
--- a/.github/workflows/update-dependencies.yml
+++ /dev/null
@@ -1,92 +0,0 @@
-name: Update dependencies
-
-on:
-  # Allow manual executions
-  workflow_dispatch:
-  # Run nightly
-  schedule:
-    - cron: '0 0 * * *'
-
-jobs:
-  propose_github_release_updates:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        include:
-          - key: kolla
-            repo_path: ${{ github.workspace }}/src/kolla
-            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
-            repository: stackhpc/kolla
-            version_jsonpath: stackhpc_kolla_source_version
-
-          - key: kolla-ansible
-            repo_path: ${{ github.workspace }}/src/kolla-ansible
-            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
-            repository: stackhpc/kolla-ansible
-            version_jsonpath: stackhpc_kolla_ansible_source_version
-
-          - key: kayobe
-            repo_path: ${{ github.workspace }}/src/kayobe
-            path: ${{ github.workspace }}/src/kayobe-config/requirements.txt
-            repository: stackhpc/kayobe
-    permissions:
-      contents: write
-      pull-requests: write
-    name: ${{ matrix.key }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-          path: ${{ github.workspace }}/src/kayobe-config
-
-      - name: Determine OpenStack release
-        id: openstack_release
-        run: |
-          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
-
-      - name: Checkout the dependency repo
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ matrix.repository }}
-          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
-          path: ${{ matrix.repo_path }}
-
-      - name: Get latest tag
-        id: latest_tag
-        run: |
-          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
-          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
-        working-directory: ${{ matrix.repo_path }}
-
-      - name: Update dependency key (kolla)
-        uses: azimuth-cloud/github-actions/config-update@master
-        with:
-          path: ${{ matrix.path }}
-          updates: |
-            ${{ matrix.version_jsonpath }}=${{ steps.latest_tag.outputs.latest_tag }}
-        if: contains(matrix.key, 'kolla')
-
-      - name: Update dependency key (kayobe)
-        run: |
-          REPLACE=$(sed -i "s/@stackhpc\/.*$/$TAG/g" $REQUIREMENTS)
-        if: contains(matrix.key, 'kayobe')
-        env:
-          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
-          REQUIREMENTS: ${{ matrix.path }}
-
-      - name: Propose changes via PR if required
-        uses: peter-evans/create-pull-request@v7
-        with:
-          token: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
-          path: ${{ github.workspace }}/src/kayobe-config
-          commit-message: >-
-            Bump ${{ matrix.key }} to ${{ steps.next.outputs.version }}
-          branch: update-dependency/${{ matrix.key }}
-          delete-branch: true
-          title: >-
-            Bump ${{ matrix.key }} to ${{ steps.next.outputs.version }}
-          body: >
-            This PR was created automatically to update
-            ${{ matrix.key }} to ${{ steps.next.outputs.version }}.
-          labels: |
-            stackhpc-ci

From a5abccf919a7a7846c0234400e6ed1b4ff12f7d9 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 19 Feb 2025 10:13:02 +0000
Subject: [PATCH 03/10] Revert workflow

---
 .github/workflows/package-build-ofed.yml | 285 +++++++++++++++++++----
 1 file changed, 238 insertions(+), 47 deletions(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 197c0a9cec..798e0c4bf2 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -1,63 +1,254 @@
-name: Update dependencies
-
+---
+name: Build OFED packages
 on:
-  # Allow manual executions
   workflow_dispatch:
+    inputs:
+      rocky9:
+        description: Build Rocky Linux 9
+        type: boolean
+        default: true
+    secrets:
+      KAYOBE_VAULT_PASSWORD:
+        required: true
+      CLOUDS_YAML:
+        required: true
+      OS_APPLICATION_CREDENTIAL_ID:
+        required: true
+      OS_APPLICATION_CREDENTIAL_SECRET:
+        required: true
 
+env:
+  ANSIBLE_FORCE_COLOR: True
+  KAYOBE_ENVIRONMENT: ci-builder
+  KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
-  propose_github_release_updates:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        include:
-          - key: kolla
-            repo_path: ${{ github.workspace }}/src/kolla
-            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
-            repository: stackhpc/kolla
-            version_jsonpath: stackhpc_kolla_source_version
-
-          - key: kolla-ansible
-            repo_path: ${{ github.workspace }}/src/kolla-ansible
-            path: ${{ github.workspace }}/src/kayobe-config/etc/kayobe/stackhpc.yml
-            repository: stackhpc/kolla-ansible
-            version_jsonpath: stackhpc_kolla_ansible_source_version
-
-          - key: kayobe
-            repo_path: ${{ github.workspace }}/src/kayobe
-            path: ${{ github.workspace }}/src/kayobe-config/requirements.txt
-            repository: stackhpc/kayobe
-    name: ${{ matrix.key }}
+  overcloud-ofed-packages:
+    name: Build OFED packages
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: arc-skc-host-image-builder-runner
+    permissions: {}
     steps:
+      - name: Install Package
+        uses: ConorMacBride/install-package@main
+        with:
+          apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
+
+      - name: Start the SSH service
+        run: |
+          sudo /etc/init.d/ssh start
+
       - name: Checkout
         uses: actions/checkout@v4
-          path: ${{ github.workspace }}/src/kayobe-config
+        with:
+          path: src/kayobe-config
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
-      - name: Checkout the dependency repo
+      - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
-          repository: ${{ matrix.repository }}
-          ref: stackhpc/2024.1
-          path: ${{ matrix.repo_path }}
+          repository: stackhpc/kayobe
+          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          path: src/kayobe
 
-      - name: Get latest tag
-        id: latest_tag
+      - name: Install Kayobe
         run: |
-          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
-          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
-        working-directory: ${{ matrix.repo_path }}
+          mkdir -p venvs &&
+          pushd venvs &&
+          python3 -m venv kayobe &&
+          source kayobe/bin/activate &&
+          pip install -U pip &&
+          pip install ../src/kayobe
 
-      - name: Update dependency key (kolla)
-        uses: azimuth-cloud/github-actions/config-update@master
-        with:
-          path: ${{ matrix.path }}
-          updates: |
-            ${{ matrix.version_jsonpath }}=${{ steps.latest_tag.outputs.latest_tag }}
-        if: contains(matrix.key, 'kolla')
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Initialise terraform
+        run: terraform init
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Generate SSH keypair
+        run: ssh-keygen -f id_rsa -N ''
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Generate clouds.yaml
+        run: |
+          cat << EOF > clouds.yaml
+          ${{ secrets.CLOUDS_YAML }}
+          EOF
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Output image tag
+        id: image_tag
+        run: |
+          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
+      - name: Output image name
+        id: image_name
+        run: |
+          echo image_name=overcloud-rocky-9-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
+
+      - name: Generate terraform.tfvars
+        run: |
+          cat << EOF > terraform.tfvars
+          ssh_public_key = "id_rsa.pub"
+          ssh_username = "cloud-user"
+          aio_vm_name = "skc-ofed-builder"
+          aio_vm_image = "${{ env.VM_IMAGE }}"
+          aio_vm_flavor = "en1.medium"
+          aio_vm_network = "stackhpc-ci"
+          aio_vm_subnet = "stackhpc-ci"
+          aio_vm_interface = "ens3"
+          EOF
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+        env:
+          VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
+
+      - name: Terraform Plan
+        run: terraform plan
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+        env:
+          OS_CLOUD: "openstack"
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Terraform Apply
+        run: |
+          for attempt in $(seq 5); do
+              if terraform apply -auto-approve; then
+                  echo "Created infrastructure on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to create infrastructure on attempt $attempt"
+              sleep 10
+              terraform destroy -auto-approve
+              sleep 60
+          done
+          echo "Failed to create infrastructure after $attempt attempts"
+          exit 1
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+        env:
+          OS_CLOUD: "openstack"
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Get Terraform outputs
+        id: tf_outputs
+        run: |
+          terraform output -json
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Write Terraform outputs
+        run: |
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
+          ${{ steps.tf_outputs.outputs.stdout }}
+          EOF
+
+      - name: Write Terraform network config
+        run: |
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
+          ---
+          aio_ips:
+            builder: "{{ access_ip_v4.value }}"
+          EOF
+
+      - name: Write Terraform network interface config
+        run: |
+          mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
+          rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
+          admin_interface: "{{ access_interface.value }}"
+          aio_interface: "{{ access_interface.value }}"
+          EOF
+
+      - name: Manage SSH keys
+        run: |
+          mkdir -p ~/.ssh
+          touch ~/.ssh/authorized_keys
+          cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
+          cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
+
+      - name: Bootstrap the control host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe control host bootstrap
+
+      - name: Run growroot playbook
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Configure the seed host (Builder VM)
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host configure --skip-tags network,docker
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Run a distro-sync
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --become --command "dnf distro-sync --refresh"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Reset BLS entries on the seed host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reset-bls-entries.yml \
+          -e "reset_bls_host=ofed-builder"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Update dependency key (kayobe)
+      - name: Disable noexec in /var/tmp
         run: |
-          REPLACE=$(sed -i "s/@stackhpc\/.*$/$TAG/g" $REQUIREMENTS)
-        if: contains(matrix.key, 'kayobe')
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --become --command "sed -i 's/noexec,//g' /etc/fstab"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Reboot to apply the kernel update
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reboot.yml
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Run OFED builder playbook
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Run OFED upload playbook
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Destroy
+        run: terraform destroy -auto-approve
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
         env:
-          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
-          REQUIREMENTS: ${{ matrix.path }}
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+        if: always()

From 2d85e6478a1029ace9eaf52e9bbc9512732f9246 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 24 Feb 2025 13:21:16 +0000
Subject: [PATCH 04/10] Reinstate workflow

---
 .github/workflows/update-dependencies.yml | 85 +++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 .github/workflows/update-dependencies.yml

diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml
new file mode 100644
index 0000000000..a37660374a
--- /dev/null
+++ b/.github/workflows/update-dependencies.yml
@@ -0,0 +1,85 @@
+name: Update dependencies
+
+on:
+  # Allow manual executions
+  workflow_dispatch:
+  # Run nightly
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  propose_github_release_updates:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - key: kolla
+            path: src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla
+            search_regex: 'stackhpc_kolla_source_version\:.*$'
+            prefix: 'stackhpc_kolla_source_version\: '
+
+          - key: kolla-ansible
+            path: src/kayobe-config/etc/kayobe/stackhpc.yml
+            repository: stackhpc/kolla-ansible
+            search_regex: 'stackhpc_kolla_ansible_source_version\:.*$'
+            prefix: 'stackhpc_kolla_ansible_source_version\: '
+
+          - key: kayobe
+            path: src/kayobe-config/requirements.txt
+            repository: stackhpc/kayobe
+            search_regex: '@stackhpc\/.*$'
+            prefix: '@'
+    permissions: write-all
+    name: ${{ matrix.key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.workspace }}/src/kayobe-config
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/kayobe-config
+
+      - name: Checkout the dependency repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.repository }}
+          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          fetch-tags: true
+          path: ${{ github.workspace }}/src/${{ matrix.key }}
+
+      - name: Get latest tag
+        id: latest_tag
+        run: |
+          TAG=$(git describe --tags --abbrev=0 --match stackhpc/\*)
+          echo latest_tag=${TAG} >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/${{ matrix.key }}
+
+      - name: Update dependency key
+        run: |
+          TAG_OVERRIDE=$(echo $TAG | sed 's/\//\\\//g')
+          sed -i "s/$SEARCH/$PREFIX$TAG_OVERRIDE/g" $REQUIREMENTS
+        env:
+          PREFIX: ${{ matrix.prefix }}
+          TAG: ${{ steps.latest_tag.outputs.latest_tag }}
+          REQUIREMENTS: ${{ github.workspace }}/${{ matrix.path }}
+          SEARCH: ${{ matrix.search_regex }}
+
+      - name: Propose changes via PR if required
+        uses: peter-evans/create-pull-request@v7
+        with:
+          path: ${{ github.workspace }}/src/kayobe-config
+          commit-message: >-
+            Bump ${{ matrix.key }} to ${{ steps.latest_tag.outputs.latest_tag }}
+          branch: update-dependency/${{ matrix.key }}
+          delete-branch: true
+          title: >-
+            Bump ${{ matrix.key }} to ${{ steps.latest_tag.outputs.latest_tag }}
+          body: >
+            This PR was created automatically to update
+            ${{ matrix.key }} to ${{ steps.latest_tag.outputs.latest_tag }}.

From 90c6000d9a2e3fa4f7dca44c804d8bda62abeca0 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 24 Feb 2025 13:25:29 +0000
Subject: [PATCH 05/10] Reno

---
 .../notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml    | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml

diff --git a/releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml b/releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml
new file mode 100644
index 0000000000..acbe45077f
--- /dev/null
+++ b/releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Workflow to update Kolla dependencies (Kayobe, Kolla and Kolla-Ansible)
+    to the latest tag available in the StackHPC branch via CI.

From c4c3456c90fb23b53ad7eeebc789d2efe35115c1 Mon Sep 17 00:00:00 2001
From: assumptionsandg <39007539+assumptionsandg@users.noreply.github.com>
Date: Mon, 24 Feb 2025 13:23:08 +0000
Subject: [PATCH 06/10] Bump kayobe to stackhpc/16.5.0.1

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index aaf9981640..cac612a768 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/2024.1
+kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/16.5.0.1
 ansible-modules-hashivault>=5.2.1
 jmespath

From 0ba32c65c17afe644d5f79fc0576f453897e729f Mon Sep 17 00:00:00 2001
From: assumptionsandg <39007539+assumptionsandg@users.noreply.github.com>
Date: Mon, 24 Feb 2025 13:23:08 +0000
Subject: [PATCH 07/10] Bump kolla to stackhpc/18.5.0.1

---
 etc/kayobe/stackhpc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 8572417a1c..f8bd139bbc 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -145,7 +145,7 @@ stackhpc_repo_elrepo_9_version: "{{ stackhpc_repo_distribution }}"
 
 # Kolla source repository.
 stackhpc_kolla_source_url: "https://github.com/stackhpc/kolla"
-stackhpc_kolla_source_version: "stackhpc/{{ openstack_release }}"
+stackhpc_kolla_source_version: stackhpc/18.5.0.1
 
 # Kolla Ansible source repository.
 stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"

From ee7f634f096bce51550f5ea447a39ac9084f7b08 Mon Sep 17 00:00:00 2001
From: assumptionsandg <39007539+assumptionsandg@users.noreply.github.com>
Date: Mon, 24 Feb 2025 13:14:16 +0000
Subject: [PATCH 08/10] Bump kolla-ansible to stackhpc/18.5.0.1

---
 etc/kayobe/stackhpc.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index f8bd139bbc..ed9b8fb79e 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -149,7 +149,7 @@ stackhpc_kolla_source_version: stackhpc/18.5.0.1
 
 # Kolla Ansible source repository.
 stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"
-stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
+stackhpc_kolla_ansible_source_version: stackhpc/18.5.0.1
 
 ###############################################################################
 # Container image registry

From c154a631a69938d49d593b9db1817ddb4c5f140c Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 24 Feb 2025 16:17:49 +0000
Subject: [PATCH 09/10] Reduce permissions

---
 .github/workflows/update-dependencies.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml
index a37660374a..a85bcfb092 100644
--- a/.github/workflows/update-dependencies.yml
+++ b/.github/workflows/update-dependencies.yml
@@ -30,7 +30,9 @@ jobs:
             repository: stackhpc/kayobe
             search_regex: '@stackhpc\/.*$'
             prefix: '@'
-    permissions: write-all
+    permissions:
+      contents: write
+      pull-requests: write
     name: ${{ matrix.key }}
     steps:
       - name: Checkout

From 5a77f931abadb05211413fa04844f09bfc3a9e46 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 27 Feb 2025 11:52:09 +0000
Subject: [PATCH 10/10] Update regex

---
 .github/workflows/update-dependencies.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml
index a85bcfb092..2b9c8bda74 100644
--- a/.github/workflows/update-dependencies.yml
+++ b/.github/workflows/update-dependencies.yml
@@ -28,8 +28,8 @@ jobs:
           - key: kayobe
             path: src/kayobe-config/requirements.txt
             repository: stackhpc/kayobe
-            search_regex: '@stackhpc\/.*$'
-            prefix: '@'
+            search_regex: 'kayobe@stackhpc\/.*$'
+            prefix: 'kayobe@'
     permissions:
       contents: write
       pull-requests: write