diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 38f55db04..8b5eebfc8 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -10,6 +10,11 @@ on:
             - LEAFCLOUD
             - SMS
             - ARCUS
+        cleanup_on_failure:
+          description: Cleanup Packer resources on failure
+          type: boolean
+          required: true
+          default: true
 
 jobs:
   openstack:
@@ -78,7 +83,7 @@ jobs:
           packer init .
 
           PACKER_LOG=1 packer build \
-          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -on-error=${{ github.event.inputs.cleanup_on_failure && 'cleanup' || 'abort' }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
           -var "source_image_name=${{ matrix.build.source_image_name }}" \
           -var "image_name=${{ matrix.build.image_name }}" \
diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml
index 6d41046ec..d4bab0f22 100644
--- a/ansible/roles/dnf_repos/defaults/main.yml
+++ b/ansible/roles/dnf_repos/defaults/main.yml
@@ -8,11 +8,13 @@ dnf_repos_filenames:
     appstream: 'Rocky-AppStream'
     crb: 'Rocky-PowerTools'
     extras: 'Rocky-Extras'
+    grafana: 'grafana'
   '9':
     baseos: 'rocky'
     appstream: 'rocky'
     crb: 'rocky'
     extras: 'rocky-extras'
+    grafana: 'grafana'
 
 dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
 
@@ -33,6 +35,9 @@ dnf_repos_default_repolist:
 - file: ceph
   name: Ceph
   base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+- file: "{{ dnf_repos_version_filenames.grafana }}"
+  name: grafana
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
 
 dnf_repos_openhpc_repolist:
 - name: OpenHPC
diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml
index c549dac53..d30d1bdff 100644
--- a/ansible/roles/pulp_site/defaults/main.yml
+++ b/ansible/roles/pulp_site/defaults/main.yml
@@ -28,6 +28,8 @@ pulp_site_rpm_info:
   subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
 - name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}"
   subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "grafana-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.grafana.timestamp[pulp_site_target_distribution_version_major].timestamp }}
+  subpath: "{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
 
 pulp_site_rpm_repo_defaults:
   remote_username: "{{ pulp_site_upstream_username }}"
diff --git a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
index 37fa04c35..f9117a26a 100644
--- a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
+++ b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250423-1606-b61e2f1a",
-        "RL9": "openhpc-RL9-250423-1606-b61e2f1a"
+        "RL8": "openhpc-RL8-250506-1259-abb6394b",
+        "RL9": "openhpc-RL9-250506-1259-abb6394b"
     }
 }
diff --git a/environments/common/files/grafana/grafana.repo.j2 b/environments/common/files/grafana/grafana.repo.j2
new file mode 100644
index 000000000..8f1aef5a7
--- /dev/null
+++ b/environments/common/files/grafana/grafana.repo.j2
@@ -0,0 +1,11 @@
+{{ ansible_managed | comment }}
+[grafana]
+baseurl = {{ appliances_pulp_url }}/pulp/content/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}
+enabled = 0
+name = grafana
+async = 1
+gpgcheck = 0
+{% if 'dnf_repos' in group_names and dnf_repos_password is defined %}
+password = {{ dnf_repos_password }}
+username = {{ dnf_repos_username }}
+{% endif %}
diff --git a/environments/common/inventory/group_vars/all/grafana.yml b/environments/common/inventory/group_vars/all/grafana.yml
index 90ef51c59..b03d16f37 100644
--- a/environments/common/inventory/group_vars/all/grafana.yml
+++ b/environments/common/inventory/group_vars/all/grafana.yml
@@ -2,7 +2,7 @@
 
 # See: https://github.com/cloudalchemy/ansible-grafana
 # for variable definitions.
-grafana_version: '9.5.21'
+grafana_version: '10.4.18'
 
 # need to copy some role defaults here so we can use in inventory:
 grafana_port: 3000
@@ -108,3 +108,6 @@ _grafana_auth_anon_cfg:
     org_role: Viewer
 grafana_auth: "{{ _grafana_auth_anon_cfg if grafana_auth_anonymous | bool else {} }}"
 _grafana_auth_is_anonymous: "{{ grafana_auth.anonymous | default({}) | length > 0 }}"
+
+# use ark or pulp for grafana as upstream packages disappear:
+grafana_yum_repo_template: "{{ appliances_repository_root }}/environments/common/files/grafana/grafana.repo.j2"
diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml
index f073a9ac6..2f31ee1a3 100644
--- a/environments/common/inventory/group_vars/all/timestamps.yml
+++ b/environments/common/inventory/group_vars/all/timestamps.yml
@@ -67,3 +67,10 @@ appliances_pulp_repos:
     '9':
       path: OpenHPC/3/updates/EL_9
       timestamp: 20241218T154614
+  grafana:
+    '8':
+      path: grafana/oss/rpm
+      timestamp: 20250505T025259
+    '9':
+      path: grafana/oss/rpm
+      timestamp: 20250505T025259