From 9e86ba2257ca380ff2d9d9f5ff8c76fd946eca97 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Mon, 13 Jul 2026 16:11:11 -0500 Subject: [PATCH 1/4] Add vp-manage-proxy-cluster-ca integration toggles. - odfSslCertificateExtractor.enabled: disable legacy SSL extraction/ACM policies - odfRamenTrustedCa.enabled: gate Ramen s3StoreProfiles CA injection - odfDrPrerequisites.caMaterialMode: trust-bundle for vp-manage PEM bundles Co-authored-by: Cursor --- Chart.yaml | 2 +- .../files/odf-dr-prerequisites-check.sh | 123 +++++++++++++----- templates/_helpers.tpl | 15 +++ templates/configmap-odf-ssl-playbooks.yaml | 2 + templates/job-odf-dr-prerequisites.yaml | 4 +- templates/job-odf-ramen-trusted-ca.yaml | 2 +- .../job-odf-ssl-certificate-extraction.yaml | 2 + .../job-odf-ssl-certificate-precheck.yaml | 2 + templates/placement-binding-odf-ssl-hub.yaml | 2 + templates/placement-binding-odf-ssl.yaml | 2 + templates/placement-odf-ssl-certificates.yaml | 2 + templates/placement-odf-ssl-hub.yaml | 2 + templates/policy-odf-managed-cluster-ssl.yaml | 2 + ...policy-odf-ssl-certificate-management.yaml | 2 + templates/rbac-odf-ramen-trusted-ca.yaml | 2 +- .../rbac-odf-ssl-certificate-precheck.yaml | 3 +- templates/rbac-odf-ssl-extractor.yaml | 2 + values.yaml | 10 ++ 18 files changed, 143 insertions(+), 38 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index f4c535d..021fcd1 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -8,4 +8,4 @@ keywords: - odf - pattern - regionaldr -version: 0.0.1 +version: 0.0.2 diff --git a/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh b/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh index 9f87832..7706db8 100755 --- a/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh +++ b/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh @@ -12,6 +12,7 @@ echo "Starting ODF DR prerequisites check..." HUB_CLUSTER="local-cluster" PRIMARY_CLUSTER="${PRIMARY_CLUSTER:-ocp-primary}" SECONDARY_CLUSTER="${SECONDARY_CLUSTER:-ocp-secondary}" +CA_MATERIAL_MODE="${CA_MATERIAL_MODE:-legacy}" KUBECONFIG_DIR="/tmp/kubeconfigs" MAX_ATTEMPTS=120 # 2 hours with 1 minute intervals SLEEP_INTERVAL=60 # 1 minute between checks @@ -147,7 +148,7 @@ check_s3_service_health() { # If still not found, try checking deployment instead if [[ $noobaa_operator_pods -eq 0 ]]; then - local noobaa_operator_deployment=$(oc --kubeconfig="$kubeconfig" get deployment -n openshift-storage --no-headers 2>/dev/null | grep -cE "noobaa-operator|noobaa.*operator" || echo "0") + local noobaa_operator_deployment=$(oc --kubeconfig="$kubeconfig" get deployment -n openshift-storage --no-headers 2>/dev/null | grep -E "noobaa-operator|noobaa.*operator" | wc -l || echo "0") noobaa_operator_deployment=$(echo "$noobaa_operator_deployment" | tr -d ' \n') if [[ $noobaa_operator_deployment -gt 0 ]]; then echo " NooBaa operator deployment found (pods may be managed by ODF operator)" @@ -276,37 +277,19 @@ check_ca_configuration() { return 0 } -# Function to check CA material completeness across all clusters -check_ca_material_completeness() { - local hub_kubeconfig="$1" - local primary_kubeconfig="$2" - local secondary_kubeconfig="$3" - - echo "Checking CA material completeness across all clusters..." - - # Extract CA bundle from each cluster - local hub_ca_bundle=$(oc --kubeconfig="$hub_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - local primary_ca_bundle=$(oc --kubeconfig="$primary_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - local secondary_ca_bundle=$(oc --kubeconfig="$secondary_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - - # Check if all CA bundles exist and have reasonable size - if [[ -z "$hub_ca_bundle" || ${#hub_ca_bundle} -lt 100 ]]; then - report_check_failure "CA material: hub (local-cluster) bundle missing or too small (${#hub_ca_bundle} chars)" - return 1 - fi - - if [[ -z "$primary_ca_bundle" || ${#primary_ca_bundle} -lt 100 ]]; then - report_check_failure "CA material: primary ($PRIMARY_CLUSTER) bundle missing or too small (${#primary_ca_bundle} chars)" - return 1 - fi +# Count PEM certificate blocks in a bundle (trust-manager / vp-manage output has no legacy comment markers). +count_pem_blocks() { + local bundle="$1" + echo "$bundle" | grep -c 'BEGIN CERTIFICATE' 2>/dev/null || echo "0" +} - if [[ -z "$secondary_ca_bundle" || ${#secondary_ca_bundle} -lt 100 ]]; then - report_check_failure "CA material: secondary ($SECONDARY_CLUSTER) bundle missing or too small (${#secondary_ca_bundle} chars)" - return 1 - fi +# Legacy opp-policy SSL extraction adds "# CA from -ca" markers per cluster contribution. +check_ca_material_legacy_markers() { + local hub_ca_bundle="$1" + local primary_ca_bundle="$2" + local secondary_ca_bundle="$3" - # Check if all CA bundles contain certificates from all three clusters - echo "๐Ÿ” Debug: Checking CA bundle contents..." + echo "๐Ÿ” Debug: Checking legacy CA bundle markers..." echo "Hub CA bundle size: ${#hub_ca_bundle} characters" echo "Primary CA bundle size: ${#primary_ca_bundle} characters" echo "Secondary CA bundle size: ${#secondary_ca_bundle} characters" @@ -314,7 +297,6 @@ check_ca_material_completeness() { echo "${hub_ca_bundle:0:500}" echo "" - # Look for hub cluster certificates if [[ "$hub_ca_bundle" != *"# CA from hub-ca"* ]]; then echo "Available markers in hub CA bundle:" echo "$hub_ca_bundle" | grep "^# CA from" || echo "No CA markers found" @@ -336,7 +318,6 @@ check_ca_material_completeness() { return 1 fi - # Look for primary cluster certificates (marker from odf-ssl-certificate-extraction.sh) if [[ "$hub_ca_bundle" != *"# CA from ${PRIMARY_CLUSTER}-ca"* ]]; then report_check_failure "CA material: hub bundle missing marker '# CA from ${PRIMARY_CLUSTER}-ca'" return 1 @@ -352,7 +333,6 @@ check_ca_material_completeness() { return 1 fi - # Look for secondary cluster certificates if [[ "$hub_ca_bundle" != *"# CA from ${SECONDARY_CLUSTER}-ca"* ]]; then report_check_failure "CA material: hub bundle missing marker '# CA from ${SECONDARY_CLUSTER}-ca'" return 1 @@ -368,6 +348,83 @@ check_ca_material_completeness() { return 1 fi + return 0 +} + +# vp-manage-proxy-cluster-ca / trust-manager: merged PEM without per-cluster comment markers. +check_ca_material_trust_bundle() { + local hub_ca_bundle="$1" + local primary_ca_bundle="$2" + local secondary_ca_bundle="$3" + local hub_pems primary_pems secondary_pems + + hub_pems=$(count_pem_blocks "$hub_ca_bundle") + primary_pems=$(count_pem_blocks "$primary_ca_bundle") + secondary_pems=$(count_pem_blocks "$secondary_ca_bundle") + + echo "๐Ÿ” Debug: trust-bundle CA material (no legacy markers expected)" + echo "Hub PEM blocks: $hub_pems; primary: $primary_pems; secondary: $secondary_pems" + + if [[ "$hub_pems" -lt 1 ]]; then + report_check_failure "CA material: hub bundle has no PEM certificates (trust-bundle mode)" + return 1 + fi + + if [[ "$primary_pems" -lt 1 ]]; then + report_check_failure "CA material: primary bundle has no PEM certificates (trust-bundle mode)" + return 1 + fi + + if [[ "$secondary_pems" -lt 1 ]]; then + report_check_failure "CA material: secondary bundle has no PEM certificates (trust-bundle mode)" + return 1 + fi + + return 0 +} + +# Function to check CA material completeness across all clusters +check_ca_material_completeness() { + local hub_kubeconfig="$1" + local primary_kubeconfig="$2" + local secondary_kubeconfig="$3" + + echo "Checking CA material completeness across all clusters (mode: ${CA_MATERIAL_MODE})..." + + # Extract CA bundle from each cluster + local hub_ca_bundle=$(oc --kubeconfig="$hub_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local primary_ca_bundle=$(oc --kubeconfig="$primary_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local secondary_ca_bundle=$(oc --kubeconfig="$secondary_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + + # Check if all CA bundles exist and have reasonable size + if [[ -z "$hub_ca_bundle" || ${#hub_ca_bundle} -lt 100 ]]; then + report_check_failure "CA material: hub (local-cluster) bundle missing or too small (${#hub_ca_bundle} chars)" + return 1 + fi + + if [[ -z "$primary_ca_bundle" || ${#primary_ca_bundle} -lt 100 ]]; then + report_check_failure "CA material: primary ($PRIMARY_CLUSTER) bundle missing or too small (${#primary_ca_bundle} chars)" + return 1 + fi + + if [[ -z "$secondary_ca_bundle" || ${#secondary_ca_bundle} -lt 100 ]]; then + report_check_failure "CA material: secondary ($SECONDARY_CLUSTER) bundle missing or too small (${#secondary_ca_bundle} chars)" + return 1 + fi + + case "$CA_MATERIAL_MODE" in + trust-bundle) + if ! check_ca_material_trust_bundle "$hub_ca_bundle" "$primary_ca_bundle" "$secondary_ca_bundle"; then + return 1 + fi + ;; + legacy|*) + if ! check_ca_material_legacy_markers "$hub_ca_bundle" "$primary_ca_bundle" "$secondary_ca_bundle"; then + return 1 + fi + ;; + esac + # Check that all CA bundles are identical (they should contain the same combined certificate data) if [[ "$hub_ca_bundle" != "$primary_ca_bundle" ]]; then report_check_failure "CA material: hub and primary cluster-proxy-ca-bundle contents differ (must be identical after trust sync)" diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 7059140..f62c694 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -44,6 +44,20 @@ {{- if not (hasKey $odf "postInstallFixesEnabled") -}}1{{- else if index $odf "postInstallFixesEnabled" -}}1{{- else -}}0{{- end -}} {{- end -}} +{{/* Ramen hub trusted-CA job: patch s3StoreProfiles from cluster-proxy-ca-bundle. Default on when enabled omitted. */}} +{{- define "rdr.odfRamenTrustedCaEnabled" -}} +{{- if ne "1" (include "rdr.odfPostInstallFixesEnabled" . | trim) -}}0{{- else -}} +{{- $cfg := .Values.odfRamenTrustedCa | default dict -}} +{{- if not (hasKey $cfg "enabled") -}}1{{- else if index $cfg "enabled" -}}1{{- else -}}0{{- end -}} +{{- end -}} +{{- end -}} + +{{/* ODF SSL extraction jobs, ACM CA policies, and spoke distribution. Default on when enabled omitted. */}} +{{- define "rdr.odfSslCertificateExtractorEnabled" -}} +{{- $cfg := .Values.odfSslCertificateExtractor | default dict -}} +{{- if not (hasKey $cfg "enabled") -}}1{{- else if index $cfg "enabled" -}}1{{- else -}}0{{- end -}} +{{- end -}} + {{/* Namespace for ODF CA post-install Jobs. */}} {{- define "rdr.clusterCaMgtNamespace" -}} {{- .Values.clusterCaMgt.namespace | default "cluster-ca-mgt" -}} @@ -81,3 +95,4 @@ checksum/odf-dr-ansible: {{ include "rdr.ansibleConfigChecksum" . | quote }} {{- define "opp.primaryClusterName" -}}{{ include "rdr.primaryClusterName" . }}{{- end -}} {{- define "opp.secondaryClusterName" -}}{{ include "rdr.secondaryClusterName" . }}{{- end -}} {{- define "opp.clusterCaMgtNamespace" -}}{{ include "rdr.clusterCaMgtNamespace" . }}{{- end -}} +{{- define "opp.odfSslCertificateExtractorEnabled" -}}{{ include "rdr.odfSslCertificateExtractorEnabled" . }}{{- end -}} diff --git a/templates/configmap-odf-ssl-playbooks.yaml b/templates/configmap-odf-ssl-playbooks.yaml index b2f3b1e..ac96c95 100644 --- a/templates/configmap-odf-ssl-playbooks.yaml +++ b/templates/configmap-odf-ssl-playbooks.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: v1 kind: ConfigMap metadata: @@ -33,3 +34,4 @@ data: {{ .Files.Get "scripts/ansible/tasks/kubeconfig.yml" | indent 4 }} tasks_kubeconfig_attempt.yml: | {{ .Files.Get "scripts/ansible/tasks/kubeconfig-attempt.yml" | indent 4 }} +{{- end }} diff --git a/templates/job-odf-dr-prerequisites.yaml b/templates/job-odf-dr-prerequisites.yaml index 9d63367..51f71ca 100644 --- a/templates/job-odf-dr-prerequisites.yaml +++ b/templates/job-odf-dr-prerequisites.yaml @@ -26,12 +26,14 @@ spec: value: {{ include "rdr.primaryClusterName" . | quote }} - name: SECONDARY_CLUSTER value: {{ include "rdr.secondaryClusterName" . | quote }} + - name: CA_MATERIAL_MODE + value: {{ (.Values.odfDrPrerequisites | default dict).caMaterialMode | default "legacy" | quote }} command: - /bin/bash - -c - | {{ include "rdr.ansibleStageOnly" . | indent 10 }} - export PRIMARY_CLUSTER SECONDARY_CLUSTER + export PRIMARY_CLUSTER SECONDARY_CLUSTER CA_MATERIAL_MODE ODF_SCRIPT="$STAGE/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh" echo "Running ODF prerequisites script with line-buffered output (kubectl logs -f will show progress)." if command -v stdbuf >/dev/null 2>&1; then diff --git a/templates/job-odf-ramen-trusted-ca.yaml b/templates/job-odf-ramen-trusted-ca.yaml index 6dd9bb0..f751dc9 100644 --- a/templates/job-odf-ramen-trusted-ca.yaml +++ b/templates/job-odf-ramen-trusted-ca.yaml @@ -1,4 +1,4 @@ -{{- if eq "1" (include "rdr.odfPostInstallFixesEnabled" . | trim) }} +{{- if eq "1" (include "rdr.odfRamenTrustedCaEnabled" . | trim) }} {{- $t := .Values.odfRamenTrustedCa | default dict }} apiVersion: batch/v1 kind: Job diff --git a/templates/job-odf-ssl-certificate-extraction.yaml b/templates/job-odf-ssl-certificate-extraction.yaml index 0217ffd..599613d 100644 --- a/templates/job-odf-ssl-certificate-extraction.yaml +++ b/templates/job-odf-ssl-certificate-extraction.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: batch/v1 kind: Job metadata: @@ -71,3 +72,4 @@ spec: serviceAccountName: odf-ssl-extractor-sa backoffLimit: 10 activeDeadlineSeconds: 7200 +{{- end }} diff --git a/templates/job-odf-ssl-certificate-precheck.yaml b/templates/job-odf-ssl-certificate-precheck.yaml index a5cb657..ff8994b 100644 --- a/templates/job-odf-ssl-certificate-precheck.yaml +++ b/templates/job-odf-ssl-certificate-precheck.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: batch/v1 kind: Job metadata: @@ -62,3 +63,4 @@ spec: path: tasks/kubeconfig.yml - key: tasks_kubeconfig_attempt.yml path: tasks/kubeconfig-attempt.yml +{{- end }} diff --git a/templates/placement-binding-odf-ssl-hub.yaml b/templates/placement-binding-odf-ssl-hub.yaml index 16ce7f1..302307f 100644 --- a/templates/placement-binding-odf-ssl-hub.yaml +++ b/templates/placement-binding-odf-ssl-hub.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: @@ -17,3 +18,4 @@ subjects: - name: policy-odf-ssl-certificate-management kind: Policy apiGroup: policy.open-cluster-management.io +{{- end }} diff --git a/templates/placement-binding-odf-ssl.yaml b/templates/placement-binding-odf-ssl.yaml index 47132c9..6d956e1 100644 --- a/templates/placement-binding-odf-ssl.yaml +++ b/templates/placement-binding-odf-ssl.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: @@ -17,3 +18,4 @@ subjects: - name: policy-odf-managed-cluster-ssl kind: Policy apiGroup: policy.open-cluster-management.io +{{- end }} diff --git a/templates/placement-odf-ssl-certificates.yaml b/templates/placement-odf-ssl-certificates.yaml index dc9a2db..c3fede5 100644 --- a/templates/placement-odf-ssl-certificates.yaml +++ b/templates/placement-odf-ssl-certificates.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: @@ -26,3 +27,4 @@ spec: - key: name operator: NotIn values: ["local-cluster"] +{{- end }} diff --git a/templates/placement-odf-ssl-hub.yaml b/templates/placement-odf-ssl-hub.yaml index 708d30d..3914677 100644 --- a/templates/placement-odf-ssl-hub.yaml +++ b/templates/placement-odf-ssl-hub.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: @@ -19,3 +20,4 @@ spec: operator: In values: - local-cluster +{{- end }} diff --git a/templates/policy-odf-managed-cluster-ssl.yaml b/templates/policy-odf-managed-cluster-ssl.yaml index 6cbdd00..7fd799a 100644 --- a/templates/policy-odf-managed-cluster-ssl.yaml +++ b/templates/policy-odf-managed-cluster-ssl.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: @@ -71,3 +72,4 @@ spec: policy.open-cluster-management.io/description: "ODF SSL certificate verification status" remediationAction: enforce severity: high +{{- end }} diff --git a/templates/policy-odf-ssl-certificate-management.yaml b/templates/policy-odf-ssl-certificate-management.yaml index 19838af..ab3c9e6 100644 --- a/templates/policy-odf-ssl-certificate-management.yaml +++ b/templates/policy-odf-ssl-certificate-management.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: policy.open-cluster-management.io/v1 kind: Policy metadata: @@ -44,3 +45,4 @@ spec: name: cluster-proxy-ca-bundle remediationAction: enforce severity: high +{{- end }} diff --git a/templates/rbac-odf-ramen-trusted-ca.yaml b/templates/rbac-odf-ramen-trusted-ca.yaml index 2cab744..58bd331 100644 --- a/templates/rbac-odf-ramen-trusted-ca.yaml +++ b/templates/rbac-odf-ramen-trusted-ca.yaml @@ -1,4 +1,4 @@ -{{- if eq "1" (include "rdr.odfPostInstallFixesEnabled" . | trim) }} +{{- if eq "1" (include "rdr.odfRamenTrustedCaEnabled" . | trim) }} # Workloads run in clusterCaMgt.namespace; cluster-proxy-ca-bundle remains in openshift-config. apiVersion: v1 kind: ServiceAccount diff --git a/templates/rbac-odf-ssl-certificate-precheck.yaml b/templates/rbac-odf-ssl-certificate-precheck.yaml index 50617cf..a16b7db 100644 --- a/templates/rbac-odf-ssl-certificate-precheck.yaml +++ b/templates/rbac-odf-ssl-certificate-precheck.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: v1 kind: ServiceAccount metadata: @@ -58,4 +59,4 @@ roleRef: subjects: - kind: ServiceAccount name: odf-ssl-certificate-precheck - namespace: {{ include "opp.clusterCaMgtNamespace" . }} \ No newline at end of file + namespace: {{ include "opp.clusterCaMgtNamespace" . }}{{- end }} diff --git a/templates/rbac-odf-ssl-extractor.yaml b/templates/rbac-odf-ssl-extractor.yaml index e6df99b..02d0747 100644 --- a/templates/rbac-odf-ssl-extractor.yaml +++ b/templates/rbac-odf-ssl-extractor.yaml @@ -1,3 +1,4 @@ +{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: v1 kind: ServiceAccount metadata: @@ -59,3 +60,4 @@ roleRef: kind: ClusterRole name: odf-ssl-extractor-role apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/values.yaml b/values.yaml index 604cecf..116d8a8 100644 --- a/values.yaml +++ b/values.yaml @@ -51,8 +51,16 @@ ansible: # -- Argo CD resource sync-options applied to the Ansible ConfigMap. configMapArgoSyncOptions: Prune=false,ServerSideApply=true +# When odf.postInstallFixesEnabled is true: odf-dr-prerequisites + odf-ramen-trusted-ca jobs/RBAC below. +odfDrPrerequisites: + # -- CA completeness check mode. legacy requires '# CA from โ€ฆ' markers from SSL extraction; + # trust-bundle validates PEM content and identical cluster-proxy-ca-bundle across clusters (vp-manage-proxy-cluster-ca). + caMaterialMode: legacy + # Post-sync job odf-ramen-trusted-ca (wave 9): CA from hub cluster-proxy-ca-bundle; waits for Ramen s3StoreProfiles. odfRamenTrustedCa: + # -- When false, skip the job that patches ramen-hub-operator-config s3StoreProfiles. + enabled: true # -- Maximum seconds to wait for Ramen s3StoreProfiles to be populated before the trusted-CA job gives up. ramenS3WaitSeconds: 3600 # -- Maximum seconds to wait for the hub cluster-proxy-ca-bundle trusted CA before the job gives up. @@ -62,6 +70,8 @@ odfRamenTrustedCa: # odf-ssl-certificate-extractor Job: wait for both DR ManagedClusters before extraction. odfSslCertificateExtractor: + # -- When false, skip SSL extraction/precheck jobs and ACM CA policies (use vp-manage-proxy-cluster-ca-chart). + enabled: true # -- Maximum number of attempts to check DR ManagedCluster readiness before the extractor job fails. clusterReadinessMaxAttempts: 150 # -- Seconds to sleep between each ManagedCluster readiness poll attempt. From 992c9b15d915d619b23f9be7a458080573db3ea8 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Mon, 3 Aug 2026 12:36:14 -0500 Subject: [PATCH 2/4] Keep ODF/MirrorPeer only; move Submariner and SSL to opp-policy. Observability storage stays here; CA trust comes from vp-manage-proxy-cluster-ca. Co-authored-by: Cursor --- .github/linters/.jscpd.json | 12 +- Chart.yaml | 7 +- README.md | 37 +- README.md.gotmpl | 9 +- ansible/playbooks/odf_ramen_trusted_ca.yml | 8 - .../playbooks/submariner_prerequisites.yml | 10 - ansible/playbooks/submariner_sg_tag.yml | 6 - .../files/odf-dr-prerequisites-check.sh | 27 +- .../odf_ramen_trusted_ca/defaults/main.yml | 6 - .../files/odf-ramen-trusted-ca.sh | 86 -- .../files/odf-ssl-ramen-hub-configmap.sh | 209 ---- .../roles/odf_ramen_trusted_ca/tasks/main.yml | 12 - .../files/submariner-prerequisites-check.sh | 198 ---- .../submariner_prerequisites/tasks/main.yml | 39 - .../files/submariner-sg-tag.sh | 557 ----------- .../roles/submariner_sg_tag/tasks/main.yml | 26 - .../odf-ssl-certificate-extraction.yml | 404 -------- scripts/ansible/odf-ssl-precheck.yml | 250 ----- scripts/ansible/requirements.yml | 5 - scripts/ansible/tasks/kubeconfig-attempt.yml | 9 - scripts/ansible/tasks/kubeconfig.yml | 84 -- scripts/ansible/tasks/odf-ssl-delete-pods.yml | 14 - scripts/ansible/tasks/odf-ssl-extract-ca.yml | 49 - .../wait-dr-managedclusters-available.yml | 53 - scripts/odf-ssl-certificate-extraction.sh | 709 -------------- scripts/odf-ssl-precheck.sh | 908 ------------------ templates/_helpers.tpl | 53 +- templates/configmap-odf-ssl-playbooks.yaml | 37 - templates/job-odf-dr-prerequisites.yaml | 8 +- templates/job-odf-ramen-trusted-ca.yaml | 66 -- .../job-odf-ssl-certificate-extraction.yaml | 75 -- .../job-odf-ssl-certificate-precheck.yaml | 66 -- templates/job-submariner-prerequisites.yaml | 51 - templates/job-submariner-sg-tag.yaml | 44 - templates/ocm-observability-bucket-claim.yaml | 13 + ... placement-binding-observability-hub.yaml} | 12 +- templates/placement-binding-odf-ssl-hub.yaml | 21 - ....yaml => placement-observability-hub.yaml} | 8 +- templates/placement-odf-ssl-certificates.yaml | 30 - templates/policy-observability-storage.yaml | 101 ++ templates/policy-odf-managed-cluster-ssl.yaml | 75 -- ...policy-odf-ssl-certificate-management.yaml | 48 - templates/rbac-odf-ramen-trusted-ca.yaml | 46 - .../rbac-odf-ssl-certificate-precheck.yaml | 62 -- templates/rbac-odf-ssl-extractor.yaml | 63 -- templates/rbac-submariner-prerequisites.yaml | 70 -- templates/rbac-submariner-sg-tag.yaml | 60 -- templates/submariner_addon_install.yaml | 66 -- values.yaml | 76 +- 49 files changed, 201 insertions(+), 4684 deletions(-) delete mode 100644 ansible/playbooks/odf_ramen_trusted_ca.yml delete mode 100644 ansible/playbooks/submariner_prerequisites.yml delete mode 100644 ansible/playbooks/submariner_sg_tag.yml delete mode 100644 ansible/roles/odf_ramen_trusted_ca/defaults/main.yml delete mode 100755 ansible/roles/odf_ramen_trusted_ca/files/odf-ramen-trusted-ca.sh delete mode 100755 ansible/roles/odf_ramen_trusted_ca/files/odf-ssl-ramen-hub-configmap.sh delete mode 100644 ansible/roles/odf_ramen_trusted_ca/tasks/main.yml delete mode 100755 ansible/roles/submariner_prerequisites/files/submariner-prerequisites-check.sh delete mode 100644 ansible/roles/submariner_prerequisites/tasks/main.yml delete mode 100755 ansible/roles/submariner_sg_tag/files/submariner-sg-tag.sh delete mode 100644 ansible/roles/submariner_sg_tag/tasks/main.yml delete mode 100644 scripts/ansible/odf-ssl-certificate-extraction.yml delete mode 100644 scripts/ansible/odf-ssl-precheck.yml delete mode 100644 scripts/ansible/requirements.yml delete mode 100644 scripts/ansible/tasks/kubeconfig-attempt.yml delete mode 100644 scripts/ansible/tasks/kubeconfig.yml delete mode 100644 scripts/ansible/tasks/odf-ssl-delete-pods.yml delete mode 100644 scripts/ansible/tasks/odf-ssl-extract-ca.yml delete mode 100644 scripts/ansible/tasks/wait-dr-managedclusters-available.yml delete mode 100755 scripts/odf-ssl-certificate-extraction.sh delete mode 100755 scripts/odf-ssl-precheck.sh delete mode 100644 templates/configmap-odf-ssl-playbooks.yaml delete mode 100644 templates/job-odf-ramen-trusted-ca.yaml delete mode 100644 templates/job-odf-ssl-certificate-extraction.yaml delete mode 100644 templates/job-odf-ssl-certificate-precheck.yaml delete mode 100644 templates/job-submariner-prerequisites.yaml delete mode 100644 templates/job-submariner-sg-tag.yaml create mode 100644 templates/ocm-observability-bucket-claim.yaml rename templates/{placement-binding-odf-ssl.yaml => placement-binding-observability-hub.yaml} (57%) delete mode 100644 templates/placement-binding-odf-ssl-hub.yaml rename templates/{placement-odf-ssl-hub.yaml => placement-observability-hub.yaml} (67%) delete mode 100644 templates/placement-odf-ssl-certificates.yaml create mode 100644 templates/policy-observability-storage.yaml delete mode 100644 templates/policy-odf-managed-cluster-ssl.yaml delete mode 100644 templates/policy-odf-ssl-certificate-management.yaml delete mode 100644 templates/rbac-odf-ramen-trusted-ca.yaml delete mode 100644 templates/rbac-odf-ssl-certificate-precheck.yaml delete mode 100644 templates/rbac-odf-ssl-extractor.yaml delete mode 100644 templates/rbac-submariner-prerequisites.yaml delete mode 100644 templates/rbac-submariner-sg-tag.yaml delete mode 100644 templates/submariner_addon_install.yaml diff --git a/.github/linters/.jscpd.json b/.github/linters/.jscpd.json index a701ce6..9633b59 100644 --- a/.github/linters/.jscpd.json +++ b/.github/linters/.jscpd.json @@ -1,13 +1,7 @@ { + "threshold": 20, "ignore": [ - "templates/_helpers.tpl", - "templates/_ansible_bootstrap.tpl", - "templates/_ansible_configmap_data.tpl", - "templates/rbac-odf-dr-prerequisites.yaml", - "templates/rbac-odf-ramen-trusted-ca.yaml", - "templates/rbac-odf-ssl-certificate-precheck.yaml", - "templates/rbac-odf-ssl-extractor.yaml", - "templates/rbac-submariner-prerequisites.yaml", - "templates/rbac-submariner-sg-tag.yaml" + "**/node_modules/**", + "templates/rbac-odf-dr-prerequisites.yaml" ] } diff --git a/Chart.yaml b/Chart.yaml index 021fcd1..f13359a 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,11 +1,10 @@ apiVersion: v2 name: odf-dr-chart description: > - Standalone Helm chart for ODF storage infrastructure supporting Regional DR. - Deploys ODF SSL certificate extraction, Submariner network overlay, MirrorPeer - storage mirroring, ODF DR prerequisites. + ODF-specific Regional DR chart: MirrorPeer, ODF prerequisites, and observability + ObjectBucketClaim/storage policy. Submariner and s3-ssl live in opp-policy-chart. keywords: - odf - pattern - regionaldr -version: 0.0.2 +version: 0.0.4 diff --git a/README.md b/README.md index b747555..f119f4e 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,19 @@ # odf-dr-chart -![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) +![Version: 0.0.4](https://img.shields.io/badge/Version-0.0.4-informational?style=flat-square) -Standalone Helm chart for ODF storage infrastructure supporting Regional DR. Deploys ODF SSL certificate extraction, Submariner network overlay, MirrorPeer storage mirroring, ODF DR prerequisites. +ODF-specific Regional DR chart: MirrorPeer, ODF prerequisites, and observability ObjectBucketClaim/storage policy. Submariner and s3-ssl live in opp-policy-chart. -This chart deploys ODF storage infrastructure supporting Regional DR: SSL certificate extraction, Submariner network overlay, MirrorPeer storage mirroring, ODF DR prerequisites, and Ramen hub trusted-CA workloads. -Consumed by the ramendr pattern as a dedicated ArgoCD application. +Always deployed with **regionaldr-with-virt**. This chart supplies ODF-specific pieces: MirrorPeer, ODF prerequisites checker, and observability ObjectBucketClaim/storage policy. +Submariner and s3-ssl live in **opp-policy-chart**. Cluster CA is owned by **vp-manage-proxy-cluster-ca**. +Ramen `s3StoreProfiles` `caCertificates` injection is owned by **opp-policy-chart** (`s3CaInjector`). ## Notable changes +v0.0.4 - Keep only ODF/MirrorPeer/observability; move Submariner and SSL to opp-policy; drop ramen trusted-CA job + +v0.0.3 - Add ObjectBucketClaim and policy-observability-storage (from opp-policy-chart) + v0.1.0 - Initial release ## Values @@ -19,24 +24,16 @@ v0.1.0 - Initial release | ansible.containerImage | string | `"quay.io/validatedpatterns/utility-container:latest"` | Container image used for Ansible post-install jobs. | | ansible.verbosity | int | `0` | Ansible-playbook verbosity level (0โ€“4). | | clusterCaMgt.createNamespace | bool | `false` | Create clusterCaMgt.namespace when installing the chart. | -| clusterCaMgt.namespace | string | `"cluster-ca-mgt"` | Namespace for ODF CA prerequisites and Ramen trusted-CA workloads. | -| global.clusterDomain | string | `"cluster.example.com"` | Base domain shared by all clusters (e.g. example.com). Used to derive per-cluster baseDomain. | -| global.clusterPlatform | string | `"AWS"` | Cloud platform type. AWS enables Hive ExternalSecret, ClusterDeployment platform.aws, Submariner gateway/credentials and SG-tag job. Use non-AWS (e.g. BareMetal) to skip those. | +| clusterCaMgt.namespace | string | `"cluster-ca-mgt"` | Namespace for ODF CA prerequisites workloads. | | odf.drCluster.primaryS3ProfileName | string | `""` | S3 profile name for the primary DRCluster CR. Required only when postInstallFixesEnabled is false. | | odf.drCluster.secondaryS3ProfileName | string | `""` | S3 profile name for the secondary DRCluster CR. Required only when postInstallFixesEnabled is false. | -| odf.postInstallFixesEnabled | bool | `true` | Enable ODF post-install automation (MirrorPeer, prerequisites checker, Ramen trusted-CA jobs/RBAC). | -| odfRamenTrustedCa.pollInterval | int | `15` | Polling interval in seconds for readiness checks inside the trusted-CA job. | -| odfRamenTrustedCa.ramenS3WaitSeconds | int | `3600` | Maximum seconds to wait for Ramen s3StoreProfiles to be populated before the trusted-CA job gives up. | -| odfRamenTrustedCa.trustedCaWaitSeconds | int | `3600` | Maximum seconds to wait for the hub cluster-proxy-ca-bundle trusted CA before the job gives up. | -| odfSslCertificateExtractor.clusterReadinessMaxAttempts | int | `150` | Maximum number of attempts to check DR ManagedCluster readiness before the extractor job fails. | -| odfSslCertificateExtractor.clusterReadinessSleepSeconds | int | `30` | Seconds to sleep between each ManagedCluster readiness poll attempt. | -| regionalDR[0].globalnetEnabled | bool | `false` | Enable Submariner Globalnet. Required when primary and secondary cluster CIDRs overlap. | -| regionalDR[0].name | string | `"resilient"` | Name of this DR pair set. Must be unique within the regionalDR list and match the ACM policy placement label. | -| submariner.NATTEnable | bool | `true` | Enable NAT traversal (NAT-T) for Submariner IPsec tunnels. | -| submariner.cableDriver | string | `"vxlan"` | Submariner cable driver (vxlan or libreswan). | -| submariner.instanceType | string | `"m5.xlarge"` | EC2 instance type for Submariner gateway nodes. | -| submariner.ipsecNatPort | int | `4500` | IPsec NAT-T UDP port used by Submariner. | -| submariner.sgTagJobEnabled | bool | `false` | Enable EC2 security group tagging job. AWS only; requires global.clusterPlatform=AWS. | +| odf.postInstallFixesEnabled | bool | `true` | Enable ODF post-install automation (MirrorPeer, prerequisites checker). | +| odfDrPrerequisites.caBundleName | string | `"vp-pattern-proxy-ca-bundle"` | CA ConfigMap name to validate (vp-manage-proxy-cluster-ca configMapName). | +| odfDrPrerequisites.caBundleNamespace | string | `"openshift-config"` | Namespace of the CA ConfigMap. | +| odfDrPrerequisites.caMaterialMode | string | `"trust-bundle"` | CA completeness check mode. Use trust-bundle with vp-manage-proxy-cluster-ca (validates PEM and identical CA ConfigMaps). legacy expects '# CA from โ€ฆ' markers. | +| regionalDR[0].clusters.primary.name | string | `"ocp-primary"` | ACM ManagedCluster name for the primary site. | +| regionalDR[0].clusters.secondary.name | string | `"ocp-secondary"` | ACM ManagedCluster name for the secondary site. | +| regionalDR[0].name | string | `"resilient"` | Name of this DR pair set. Must be unique within the regionalDR list. | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/README.md.gotmpl b/README.md.gotmpl index 91bb0eb..29cc948 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -5,11 +5,16 @@ {{ template "chart.description" . }} -This chart deploys ODF storage infrastructure supporting Regional DR: SSL certificate extraction, Submariner network overlay, MirrorPeer storage mirroring, ODF DR prerequisites, and Ramen hub trusted-CA workloads. -Consumed by the ramendr pattern as a dedicated ArgoCD application. +Always deployed with **regionaldr-with-virt**. This chart supplies ODF-specific pieces: MirrorPeer, ODF prerequisites checker, and observability ObjectBucketClaim/storage policy. +Submariner and s3-ssl live in **opp-policy-chart**. Cluster CA is owned by **vp-manage-proxy-cluster-ca**. +Ramen `s3StoreProfiles` `caCertificates` injection is owned by **opp-policy-chart** (`s3CaInjector`). ## Notable changes +v0.0.4 - Keep only ODF/MirrorPeer/observability; move Submariner and SSL to opp-policy; drop ramen trusted-CA job + +v0.0.3 - Add ObjectBucketClaim and policy-observability-storage (from opp-policy-chart) + v0.1.0 - Initial release {{ template "chart.homepageLine" . }} diff --git a/ansible/playbooks/odf_ramen_trusted_ca.yml b/ansible/playbooks/odf_ramen_trusted_ca.yml deleted file mode 100644 index 34f43fb..0000000 --- a/ansible/playbooks/odf_ramen_trusted_ca.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Runs on hub after MirrorPeer (sync wave): use cluster-proxy-ca-bundle CA material, wait for -# s3StoreProfiles in ramen-hub-operator-config, then patch caCertificates (opp-policy-chart parity). -- name: Ramen hub CA from trusted bundle - hosts: localhost - gather_facts: false - roles: - - odf_ramen_trusted_ca diff --git a/ansible/playbooks/submariner_prerequisites.yml b/ansible/playbooks/submariner_prerequisites.yml deleted file mode 100644 index 15b48b5..0000000 --- a/ansible/playbooks/submariner_prerequisites.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Submariner prerequisites - hosts: localhost - gather_facts: false - vars: - primary_cluster: "{{ lookup('env', 'PRIMARY_CLUSTER') | default('ocp-primary', true) }}" - secondary_cluster: "{{ lookup('env', 'SECONDARY_CLUSTER') | default('ocp-secondary', true) }}" - submariner_broker_namespace: "{{ lookup('env', 'SUBMARINER_BROKER_NAMESPACE') | default('resilient-broker', true) }}" - roles: - - submariner_prerequisites diff --git a/ansible/playbooks/submariner_sg_tag.yml b/ansible/playbooks/submariner_sg_tag.yml deleted file mode 100644 index a4d3662..0000000 --- a/ansible/playbooks/submariner_sg_tag.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Submariner security group AWS tagging - hosts: localhost - gather_facts: false - roles: - - submariner_sg_tag diff --git a/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh b/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh index 7706db8..d4a4f4c 100755 --- a/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh +++ b/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh @@ -6,6 +6,9 @@ if [[ -z "${ODF_PREREQ_LINEBUF:-}" ]] && command -v stdbuf >/dev/null 2>&1; then fi set -euo pipefail +CA_BUNDLE_NAME="$CA_BUNDLE_NAME" +CA_BUNDLE_NAMESPACE="${CA_BUNDLE_NAMESPACE:-openshift-config}" + echo "Starting ODF DR prerequisites check..." # Configuration (PRIMARY_CLUSTER and SECONDARY_CLUSTER from values.yaml via env) @@ -252,24 +255,24 @@ check_ca_configuration() { echo "Checking CA configuration on $cluster..." - # Check if cluster-proxy-ca-bundle ConfigMap exists - if ! oc --kubeconfig="$kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config &>/dev/null; then - report_check_failure "CA config ($cluster): ConfigMap cluster-proxy-ca-bundle not found in openshift-config" + # Check if $CA_BUNDLE_NAME ConfigMap exists + if ! oc --kubeconfig="$kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" &>/dev/null; then + report_check_failure "CA config ($cluster): ConfigMap $CA_BUNDLE_NAME not found in $CA_BUNDLE_NAMESPACE" return 1 fi # Check if ConfigMap has certificate data - local ca_bundle_size=$(oc --kubeconfig="$kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c || echo "0") + local ca_bundle_size=$(oc --kubeconfig="$kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c || echo "0") ca_bundle_size=$(echo "$ca_bundle_size" | tr -d ' \n') if [[ $ca_bundle_size -lt 100 ]]; then - report_check_failure "CA config ($cluster): cluster-proxy-ca-bundle data ca-bundle.crt too small or empty (bytes: $ca_bundle_size)" + report_check_failure "CA config ($cluster): $CA_BUNDLE_NAME data ca-bundle.crt too small or empty (bytes: $ca_bundle_size)" return 1 fi # Check if Proxy object is configured local proxy_trusted_ca=$(oc --kubeconfig="$kubeconfig" get proxy cluster -o jsonpath='{.spec.trustedCA.name}' 2>/dev/null || echo "") - if [[ "$proxy_trusted_ca" != "cluster-proxy-ca-bundle" ]]; then - report_check_failure "CA config ($cluster): Proxy cluster spec.trustedCA.name is '$proxy_trusted_ca' (expected cluster-proxy-ca-bundle)" + if [[ "$proxy_trusted_ca" != "$CA_BUNDLE_NAME" ]]; then + report_check_failure "CA config ($cluster): Proxy cluster spec.trustedCA.name is '$proxy_trusted_ca' (expected $CA_BUNDLE_NAME)" return 1 fi @@ -392,9 +395,9 @@ check_ca_material_completeness() { echo "Checking CA material completeness across all clusters (mode: ${CA_MATERIAL_MODE})..." # Extract CA bundle from each cluster - local hub_ca_bundle=$(oc --kubeconfig="$hub_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - local primary_ca_bundle=$(oc --kubeconfig="$primary_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - local secondary_ca_bundle=$(oc --kubeconfig="$secondary_kubeconfig" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local hub_ca_bundle=$(oc --kubeconfig="$hub_kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local primary_ca_bundle=$(oc --kubeconfig="$primary_kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local secondary_ca_bundle=$(oc --kubeconfig="$secondary_kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") # Check if all CA bundles exist and have reasonable size if [[ -z "$hub_ca_bundle" || ${#hub_ca_bundle} -lt 100 ]]; then @@ -427,12 +430,12 @@ check_ca_material_completeness() { # Check that all CA bundles are identical (they should contain the same combined certificate data) if [[ "$hub_ca_bundle" != "$primary_ca_bundle" ]]; then - report_check_failure "CA material: hub and primary cluster-proxy-ca-bundle contents differ (must be identical after trust sync)" + report_check_failure "CA material: hub and primary $CA_BUNDLE_NAME contents differ (must be identical after trust sync)" return 1 fi if [[ "$hub_ca_bundle" != "$secondary_ca_bundle" ]]; then - report_check_failure "CA material: hub and secondary cluster-proxy-ca-bundle contents differ (must be identical after trust sync)" + report_check_failure "CA material: hub and secondary $CA_BUNDLE_NAME contents differ (must be identical after trust sync)" return 1 fi diff --git a/ansible/roles/odf_ramen_trusted_ca/defaults/main.yml b/ansible/roles/odf_ramen_trusted_ca/defaults/main.yml deleted file mode 100644 index 8ccdae8..0000000 --- a/ansible/roles/odf_ramen_trusted_ca/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -odf_ramen_trusted_ca_work_dir: /tmp/odf-ssl-certs -# Waits: hub trusted CA bundle (from policy) and Ramen s3StoreProfiles (from ODF after MirrorPeer) -odf_ramen_trusted_ca_wait_seconds: 3600 -odf_ramen_s3_wait_seconds: 3600 -odf_ramen_poll_interval: 15 diff --git a/ansible/roles/odf_ramen_trusted_ca/files/odf-ramen-trusted-ca.sh b/ansible/roles/odf_ramen_trusted_ca/files/odf-ramen-trusted-ca.sh deleted file mode 100755 index 021c036..0000000 --- a/ansible/roles/odf_ramen_trusted_ca/files/odf-ramen-trusted-ca.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# After MirrorPeer (and policies that populate cluster-proxy-ca-bundle), copy CA from the hub -# cluster-proxy-ca-bundle ConfigMap โ€” do not re-extract from router/spoke API servers. -# Wait until Ramen hub config has s3StoreProfiles (from ODF/MirrorPeer), then patch caCertificates only. -set -euo pipefail - -PRIMARY_CLUSTER="${PRIMARY_CLUSTER:?PRIMARY_CLUSTER is required}" -SECONDARY_CLUSTER="${SECONDARY_CLUSTER:?SECONDARY_CLUSTER is required}" -WORK_DIR="${WORK_DIR:-/tmp/odf-ssl-certs}" -RAMEN_CM_WAIT_SECONDS="${RAMEN_CM_WAIT_SECONDS:-3600}" -TRUSTED_CA_WAIT_SECONDS="${TRUSTED_CA_WAIT_SECONDS:-3600}" -POLL_INTERVAL="${POLL_INTERVAL:-15}" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RAMEN_SCRIPT="${SCRIPT_DIR}/odf-ssl-ramen-hub-configmap.sh" - -die() { - echo "โŒ odf-ramen-trusted-ca.sh: $*" >&2 - exit 1 -} - -command -v oc >/dev/null 2>&1 || die "oc not found" -[[ -x "$RAMEN_SCRIPT" ]] || [[ -f "$RAMEN_SCRIPT" ]] || die "missing $RAMEN_SCRIPT" -chmod +x "$RAMEN_SCRIPT" 2>/dev/null || true - -mkdir -p "$WORK_DIR" - -wait_for_trusted_ca() { - local deadline=$((SECONDS + TRUSTED_CA_WAIT_SECONDS)) - echo "Waiting for cluster-proxy-ca-bundle (openshift-config) with non-trivial ca-bundle.crt (max ${TRUSTED_CA_WAIT_SECONDS}s)..." - while ((SECONDS < deadline)); do - local data bytes - data=$(oc get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || true) - bytes=$(printf '%s' "$data" | wc -c | tr -d ' ') - if [[ "${bytes:-0}" -ge 64 ]]; then - printf '%s' "$data" >"$WORK_DIR/combined-ca-bundle.crt" - echo " โœ… trusted CA bundle captured (${bytes} bytes)" - return 0 - fi - echo " ... ca-bundle.crt bytes=${bytes:-0}, retry in ${POLL_INTERVAL}s" - sleep "$POLL_INTERVAL" - done - die "cluster-proxy-ca-bundle not ready in time โ€” ensure ACM/ODF policy populated it (see opp-policy-chart policy-odf-managed-cluster-ssl)" -} - -count_s3_profiles() { - local yaml="$1" - [[ -n "$yaml" ]] || { - echo 0 - return - } - if command -v yq &>/dev/null; then - local k t - k=$(echo "$yaml" | yq eval '.kubeObjectProtection.s3StoreProfiles | length' 2>/dev/null | tr -d ' \n\r' || echo 0) - t=$(echo "$yaml" | yq eval '.s3StoreProfiles | length' 2>/dev/null | tr -d ' \n\r' || echo 0) - k=$((10#${k:-0})) - t=$((10#${t:-0})) - echo $((k > t ? k : t)) - else - echo "$yaml" | grep -c 's3ProfileName:' 2>/dev/null || echo 0 - fi -} - -wait_for_ramen_s3_profiles() { - local deadline=$((SECONDS + RAMEN_CM_WAIT_SECONDS)) yaml c - echo "Waiting for ramen-hub-operator-config s3StoreProfiles (openshift-operators, max ${RAMEN_CM_WAIT_SECONDS}s)..." - while ((SECONDS < deadline)); do - yaml=$(oc get configmap ramen-hub-operator-config -n openshift-operators -o jsonpath='{.data.ramen_manager_config\.yaml}' 2>/dev/null || true) - if [[ -n "$yaml" ]] && echo "$yaml" | grep -q 's3StoreProfiles'; then - c=$(count_s3_profiles "$yaml") - if [[ "${c:-0}" -ge 2 ]]; then - echo " โœ… ramen_manager_config has s3StoreProfiles (countโ‰ˆ$c)" - return 0 - fi - fi - echo " ... profiles not ready yet (need >=2), retry in ${POLL_INTERVAL}s" - sleep "$POLL_INTERVAL" - done - die "ramen-hub-operator-config never gained s3StoreProfiles โ€” confirm MirrorPeer and hub Ramen operator reconciled" -} - -wait_for_trusted_ca -wait_for_ramen_s3_profiles - -export WORK_DIR PRIMARY_CLUSTER SECONDARY_CLUSTER -exec bash "$RAMEN_SCRIPT" diff --git a/ansible/roles/odf_ramen_trusted_ca/files/odf-ssl-ramen-hub-configmap.sh b/ansible/roles/odf_ramen_trusted_ca/files/odf-ssl-ramen-hub-configmap.sh deleted file mode 100755 index 6f81a5e..0000000 --- a/ansible/roles/odf_ramen_trusted_ca/files/odf-ssl-ramen-hub-configmap.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env bash -# Update ramen-hub-operator-config with caCertificates in s3StoreProfiles (same logic path as -# odf-ssl-certificate-extraction.sh ยง7b). Invoked by the Ansible extraction playbook so behavior -# matches the proven shell implementation. -set -euo pipefail - -WORK_DIR="${WORK_DIR:-/tmp/odf-ssl-certs}" -PRIMARY_CLUSTER="${PRIMARY_CLUSTER:?PRIMARY_CLUSTER is required}" -SECONDARY_CLUSTER="${SECONDARY_CLUSTER:?SECONDARY_CLUSTER is required}" - -die() { - echo "โŒ odf-ssl-ramen-hub-configmap.sh: $*" >&2 - exit 1 -} - -trap 'echo "โŒ odf-ssl-ramen-hub-configmap.sh: command failed (exit $?) at line $LINENO โ€” see stderr above for the failing command." >&2' ERR - -mkdir -p "$WORK_DIR" -[[ -f "$WORK_DIR/combined-ca-bundle.crt" ]] || die "missing $WORK_DIR/combined-ca-bundle.crt" - -echo "7b. Updating ramen-hub-operator-config in openshift-operators namespace (bash parity script)..." - -CA_BUNDLE_BASE64=$(base64 -w 0 <"$WORK_DIR/combined-ca-bundle.crt" 2>/dev/null || base64 <"$WORK_DIR/combined-ca-bundle.crt" | tr -d '\n') - -# Post-apply: fetch live YAML to disk (no huge shell vars) and validate structure. -verify_post_apply() { - local f="$WORK_DIR/.ramen-post-apply-verify.yaml" attempt - local MIN_REQUIRED_PROFILES=2 - local PK PT CK CT bad maxp - local last_PK=0 last_PT=0 last_CK=0 last_CT=0 last_maxp=0 last_bad=1 oc_ok=0 - for attempt in $(seq 1 10); do - if [[ "$attempt" -gt 1 ]]; then - sleep 6 - else - sleep 2 - fi - if oc get configmap ramen-hub-operator-config -n openshift-operators \ - -o jsonpath='{.data.ramen_manager_config\.yaml}' >"$f" 2>/dev/null; then - oc_ok=1 - else - oc_ok=0 - continue - fi - [[ -s "$f" ]] || continue - grep -q 'caCertificates' "$f" || continue - grep -q 's3StoreProfiles' "$f" || continue - PK=$(yq eval '(.kubeObjectProtection.s3StoreProfiles // []) | length' "$f" 2>/dev/null | tr -d ' \n\r' | head -1 || echo 0) - PT=$(yq eval '(.s3StoreProfiles // []) | length' "$f" 2>/dev/null | tr -d ' \n\r' | head -1 || echo 0) - CK=$(yq eval '[(.kubeObjectProtection.s3StoreProfiles // [])[]? | select(has("caCertificates"))] | length' "$f" 2>/dev/null | tr -d ' \n\r' | head -1 || echo 0) - CT=$(yq eval '[(.s3StoreProfiles // [])[]? | select(has("caCertificates"))] | length' "$f" 2>/dev/null | tr -d ' \n\r' | head -1 || echo 0) - [[ "$PK" =~ ^[0-9]+$ ]] || PK=0 - [[ "$PT" =~ ^[0-9]+$ ]] || PT=0 - [[ "$CK" =~ ^[0-9]+$ ]] || CK=0 - [[ "$CT" =~ ^[0-9]+$ ]] || CT=0 - bad=0 - [[ "$PK" -gt 0 && "$CK" -lt "$PK" ]] && bad=1 - [[ "$PT" -gt 0 && "$CT" -lt "$PT" ]] && bad=1 - maxp=$((PK > PT ? PK : PT)) - last_PK=$PK - last_PT=$PT - last_CK=$CK - last_CT=$CT - last_maxp=$maxp - last_bad=$bad - [[ "$bad" -eq 1 ]] && continue - [[ "$maxp" -ge "$MIN_REQUIRED_PROFILES" ]] || continue - [[ "$CK" -ge "$MIN_REQUIRED_PROFILES" || "$CT" -ge "$MIN_REQUIRED_PROFILES" ]] || continue - echo " โœ… ramen-hub-operator-config verified (attempt $attempt): kubeObjectProtection s3 profiles $PK/$CK, top-level $PT/$CT" - return 0 - done - echo " โŒ Post-apply verification failed after 10 attempts." >&2 - echo " โŒ Diagnosis: oc_get_ok=$oc_ok last_kop_profiles=$last_PK last_kop_with_ca=$last_CK last_top_profiles=$last_PT last_top_with_ca=$last_CT last_max_profiles=$last_maxp last_section_bad=$last_bad (need each non-empty section fully CA-populated; max profiles >= $MIN_REQUIRED_PROFILES; at least $MIN_REQUIRED_PROFILES with CA in kop OR top)" >&2 - echo " โŒ If kop/top counts are 0, the hub operator may have removed ramen_manager_config data or the key is empty." >&2 - [[ -f "$f" ]] && { - echo " โŒ First 80 lines of live ramen_manager_config from cluster:" >&2 - head -n 80 "$f" >&2 - } || echo " โŒ No verify file (oc get may have failed every attempt)." >&2 - return 1 -} - -UPDATED_YAML="" - -if oc get configmap ramen-hub-operator-config -n openshift-operators &>/dev/null; then - echo " ConfigMap exists, updating ramen_manager_config.yaml with caCertificates in s3StoreProfiles..." - - EXISTING_YAML=$(oc get configmap ramen-hub-operator-config -n openshift-operators -o jsonpath='{.data.ramen_manager_config\.yaml}' 2>/dev/null || echo "") - - MIN_REQUIRED_PROFILES=2 - if [[ -n "$EXISTING_YAML" ]]; then - if command -v yq &>/dev/null; then - COUNT_KOP=$(echo "$EXISTING_YAML" | yq eval '.kubeObjectProtection.s3StoreProfiles | length' 2>/dev/null | tr -d ' \n\r' | head -1 || echo "0") - COUNT_TOP=$(echo "$EXISTING_YAML" | yq eval '.s3StoreProfiles | length' 2>/dev/null | tr -d ' \n\r' | head -1 || echo "0") - COUNT_KOP=$((10#${COUNT_KOP:-0})) - COUNT_TOP=$((10#${COUNT_TOP:-0})) - EXISTING_PROFILE_COUNT=$((COUNT_KOP >= COUNT_TOP ? COUNT_KOP : COUNT_TOP)) - else - EXISTING_PROFILE_COUNT=$(echo "$EXISTING_YAML" | grep -c "s3ProfileName:" 2>/dev/null || echo "0") - if [[ $EXISTING_PROFILE_COUNT -eq 0 ]]; then - EXISTING_PROFILE_COUNT=$(echo "$EXISTING_YAML" | grep -c "s3Bucket:" 2>/dev/null || echo "0") - fi - fi - EXISTING_PROFILE_COUNT=$(echo "$EXISTING_PROFILE_COUNT" | tr -d ' \n\r' | grep -E '^[0-9]+$' || echo "0") - EXISTING_PROFILE_COUNT=$((10#$EXISTING_PROFILE_COUNT)) - if [[ $EXISTING_PROFILE_COUNT -lt $MIN_REQUIRED_PROFILES ]]; then - echo " โŒ CRITICAL: Insufficient s3StoreProfiles in existing ConfigMap (found $EXISTING_PROFILE_COUNT, need $MIN_REQUIRED_PROFILES)" - echo "$EXISTING_YAML" | head -n 50 - die "Insufficient s3StoreProfiles โ€” pre-create profiles in ramen-hub-operator-config or use the full extraction job on the hub" - fi - echo " โœ… Found $EXISTING_PROFILE_COUNT s3StoreProfiles (patching caCertificates only)" - fi - - PATCHED_VIA_YQ=false - if [[ -n "$EXISTING_YAML" ]]; then - echo "$EXISTING_YAML" >"$WORK_DIR/existing-ramen-config.yaml" - if ! command -v yq &>/dev/null; then - die "yq is required (e.g. mikefarah/yq v4)" - fi - export CA_BUNDLE_BASE64 - YQ_PATCHED=false - if yq eval -i '.s3StoreProfiles[]? |= . + {"caCertificates": strenv(CA_BUNDLE_BASE64)}' "$WORK_DIR/existing-ramen-config.yaml" 2>/dev/null; then - YQ_PATCHED=true - fi - if yq eval -i '.kubeObjectProtection.s3StoreProfiles[]? |= . + {"caCertificates": strenv(CA_BUNDLE_BASE64)}' "$WORK_DIR/existing-ramen-config.yaml" 2>/dev/null; then - YQ_PATCHED=true - fi - if [[ "$YQ_PATCHED" != "true" ]]; then - die "yq could not patch s3StoreProfiles (check kubeObjectProtection / top-level s3StoreProfiles). yq: $(yq --version 2>/dev/null || true)" - fi - grep -q "caCertificates" "$WORK_DIR/existing-ramen-config.yaml" || die "patched file has no caCertificates" - cp "$WORK_DIR/existing-ramen-config.yaml" "$WORK_DIR/ramen_manager_config.yaml" - PATCHED_VIA_YQ=true - else - UPDATED_YAML="kubeObjectProtection: - s3StoreProfiles: - - s3ProfileName: $PRIMARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\" - - s3ProfileName: $SECONDARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\" -s3StoreProfiles: - - s3ProfileName: $PRIMARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\" - - s3ProfileName: $SECONDARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\"" - fi - - if [[ "$PATCHED_VIA_YQ" != "true" ]]; then - echo "$UPDATED_YAML" >"$WORK_DIR/ramen_manager_config.yaml" - fi - - echo " Building ConfigMap manifest (literal block) and oc apply..." - oc get configmap ramen-hub-operator-config -n openshift-operators -o yaml >"$WORK_DIR/ramen-configmap-template.yaml" 2>/dev/null || true - - UPDATE_EXIT_CODE=1 - UPDATE_OUTPUT="" - if [[ -f "$WORK_DIR/ramen-configmap-template.yaml" ]]; then - { - echo "apiVersion: v1" - echo "kind: ConfigMap" - echo "metadata:" - echo " name: ramen-hub-operator-config" - echo " namespace: openshift-operators" - echo "data:" - echo " ramen_manager_config.yaml: |" - sed 's/^/ /' "$WORK_DIR/ramen_manager_config.yaml" - } >"$WORK_DIR/ramen-configmap-updated.yaml" - if UPDATE_OUTPUT=$(oc apply -f "$WORK_DIR/ramen-configmap-updated.yaml" 2>&1); then - UPDATE_EXIT_CODE=0 - else - UPDATE_EXIT_CODE=$? - fi - rm -f "$WORK_DIR/ramen-configmap-template.yaml" "$WORK_DIR/ramen-configmap-updated.yaml" - else - if UPDATE_OUTPUT=$(oc set data configmap/ramen-hub-operator-config -n openshift-operators \ - ramen_manager_config.yaml="$(cat "$WORK_DIR/ramen_manager_config.yaml")" 2>&1); then - UPDATE_EXIT_CODE=0 - else - UPDATE_EXIT_CODE=$? - fi - fi - - echo " Update exit code: $UPDATE_EXIT_CODE" - echo " Update output: $UPDATE_OUTPUT" - - if [[ $UPDATE_EXIT_CODE -eq 0 ]]; then - verify_post_apply || die "Post-apply verification failed โ€” CA not present in live ConfigMap" - else - die "oc apply/set data failed: $UPDATE_OUTPUT" - fi - - rm -f "$WORK_DIR/existing-ramen-config.yaml" "$WORK_DIR/ramen_manager_config.yaml" || true -else - echo " ConfigMap does not exist; creating ramen-hub-operator-config..." - oc create configmap ramen-hub-operator-config -n openshift-operators \ - --from-literal=ramen_manager_config.yaml="kubeObjectProtection: - s3StoreProfiles: - - s3ProfileName: $PRIMARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\" - - s3ProfileName: $SECONDARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\" -s3StoreProfiles: - - s3ProfileName: $PRIMARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\" - - s3ProfileName: $SECONDARY_CLUSTER - caCertificates: \"$CA_BUNDLE_BASE64\"" || die "oc create configmap failed" - - verify_post_apply || die "Post-create verification failed" -fi - -echo " ramen-hub-operator-config updated successfully with base64-encoded CA bundle in s3StoreProfiles" diff --git a/ansible/roles/odf_ramen_trusted_ca/tasks/main.yml b/ansible/roles/odf_ramen_trusted_ca/tasks/main.yml deleted file mode 100644 index df2232f..0000000 --- a/ansible/roles/odf_ramen_trusted_ca/tasks/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Apply Ramen hub CA from cluster-proxy-ca-bundle (after MirrorPeer / s3 profiles) - ansible.builtin.command: - argv: - - bash - - "{{ role_path }}/files/odf-ramen-trusted-ca.sh" - environment: - WORK_DIR: "{{ odf_ramen_trusted_ca_work_dir }}" - RAMEN_CM_WAIT_SECONDS: "{{ odf_ramen_s3_wait_seconds | string }}" - TRUSTED_CA_WAIT_SECONDS: "{{ odf_ramen_trusted_ca_wait_seconds | string }}" - POLL_INTERVAL: "{{ odf_ramen_poll_interval | string }}" - changed_when: true diff --git a/ansible/roles/submariner_prerequisites/files/submariner-prerequisites-check.sh b/ansible/roles/submariner_prerequisites/files/submariner-prerequisites-check.sh deleted file mode 100755 index 4a94c70..0000000 --- a/ansible/roles/submariner_prerequisites/files/submariner-prerequisites-check.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/bash -if [[ -z "${SUBMARINER_PREREQ_LINEBUF:-}" ]] && command -v stdbuf >/dev/null 2>&1; then - export SUBMARINER_PREREQ_LINEBUF=1 - exec stdbuf -oL -eL bash "$0" "$@" -fi -set -euo pipefail - -echo "Starting Submariner prerequisites check..." - -# Configuration (PRIMARY_CLUSTER and SECONDARY_CLUSTER from values.yaml via env) -PRIMARY_CLUSTER="${PRIMARY_CLUSTER:-ocp-primary}" -SECONDARY_CLUSTER="${SECONDARY_CLUSTER:-ocp-secondary}" -SUBMARINER_BROKER_NAMESPACE="${SUBMARINER_BROKER_NAMESPACE:-resilient-broker}" -KUBECONFIG_DIR="/tmp/kubeconfigs" -MAX_ATTEMPTS=120 # 2 hours with 1 minute intervals -SLEEP_INTERVAL=60 # 1 minute between checks - -# Create kubeconfig directory -mkdir -p "$KUBECONFIG_DIR" - -progress_sleep() { - local total=${1:-60} - local step=15 - local elapsed=0 - echo "โณ Pausing ${total}s before continuing..." - while ((elapsed < total)); do - local chunk=$step - ((elapsed + chunk > total)) && chunk=$((total - elapsed)) - sleep "$chunk" - elapsed=$((elapsed + chunk)) - ((elapsed < total)) && echo " ... ${elapsed}s / ${total}s elapsed (still in wait)" - done -} - -# Function to check Submariner health and connectivity -check_submariner_health() { - local cluster="$1" - local kubeconfig="$2" - - echo "Checking Submariner health on $cluster..." - - # Check if Submariner is installed (check for the correct CRDs) - if ! oc --kubeconfig="$kubeconfig" get crd clusters.submariner.io &>/dev/null; then - echo "Submariner clusters CRD not found on $cluster" - return 1 - fi - - # Check if Submariner operator is running - local submariner_operator_pods=$(oc --kubeconfig="$kubeconfig" get pods -n submariner-operator --no-headers 2>/dev/null | grep -c "Running" || echo "0") - submariner_operator_pods=$(echo "$submariner_operator_pods" | tr -d ' \n') - if [[ $submariner_operator_pods -eq 0 ]]; then - echo "Submariner operator not running on $cluster" - return 1 - fi - - # Check Submariner gateway nodes - local gateway_nodes=$(oc --kubeconfig="$kubeconfig" get nodes -l submariner.io/gateway=true --no-headers 2>/dev/null | wc -l) - if [[ $gateway_nodes -eq 0 ]]; then - echo "No Submariner gateway nodes found on $cluster" - return 1 - fi - - echo "Submariner is healthy on $cluster" - return 0 -} - -# Function to check Submariner connectivity between clusters -check_submariner_connectivity() { - echo "Checking Submariner connectivity between $PRIMARY_CLUSTER and $SECONDARY_CLUSTER..." - - # Check Submariner clusters on hub cluster - local primary_cluster_id=$(oc get clusters.submariner.io "$PRIMARY_CLUSTER" -n "$SUBMARINER_BROKER_NAMESPACE" -o jsonpath='{.spec.cluster_id}' 2>/dev/null || echo "") - local secondary_cluster_id=$(oc get clusters.submariner.io "$SECONDARY_CLUSTER" -n "$SUBMARINER_BROKER_NAMESPACE" -o jsonpath='{.spec.cluster_id}' 2>/dev/null || echo "") - - if [[ -z "$primary_cluster_id" || -z "$secondary_cluster_id" ]]; then - echo "Could not retrieve cluster IDs from Submariner" - return 1 - fi - - echo "Primary cluster ID: $primary_cluster_id" - echo "Secondary cluster ID: $secondary_cluster_id" - - # Check if both clusters are registered in Submariner - if [[ "$primary_cluster_id" == "$PRIMARY_CLUSTER" && "$secondary_cluster_id" == "$SECONDARY_CLUSTER" ]]; then - echo "โœ… Submariner connectivity verified between $PRIMARY_CLUSTER and $SECONDARY_CLUSTER" - return 0 - else - echo "โŒ Cluster IDs do not match expected values" - return 1 - fi -} - -# Function to download kubeconfig for a cluster -download_kubeconfig() { - local cluster="$1" - local kubeconfig_path="$KUBECONFIG_DIR/${cluster}-kubeconfig.yaml" - - echo "Downloading kubeconfig for $cluster..." - - # Get the kubeconfig secret name (same approach as download-kubeconfigs.sh) - local kubeconfig_secret=$(oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1) - - if [[ -z "$kubeconfig_secret" ]]; then - echo "No kubeconfig secret found for cluster $cluster" - return 1 - fi - - echo "Found kubeconfig secret: $kubeconfig_secret" - - # Try to get the kubeconfig data (same approach as download-kubeconfigs.sh) - local kubeconfig_data="" - - # First try to get the 'kubeconfig' field - kubeconfig_data=$(oc get "$kubeconfig_secret" -n "$cluster" -o jsonpath='{.data.kubeconfig}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - - # If that fails, try the 'raw-kubeconfig' field - if [[ -z "$kubeconfig_data" ]]; then - kubeconfig_data=$(oc get "$kubeconfig_secret" -n "$cluster" -o jsonpath='{.data.raw-kubeconfig}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - fi - - if [[ -z "$kubeconfig_data" ]]; then - echo "Could not extract kubeconfig data for cluster $cluster" - return 1 - fi - - # Write the kubeconfig to file - echo "$kubeconfig_data" >"$kubeconfig_path" - - # Validate kubeconfig - if oc --kubeconfig="$kubeconfig_path" get nodes --request-timeout=5s &>/dev/null; then - echo "Kubeconfig downloaded and validated for $cluster" - return 0 - else - echo "Kubeconfig for $cluster is invalid or cluster is unreachable" - return 1 - fi -} - -# Main check loop - keep retrying until all prerequisites are met -while true; do - attempt=1 - echo "=== Starting new Submariner prerequisites check cycle ===" - - while [[ $attempt -le $MAX_ATTEMPTS ]]; do - echo "=== Submariner Prerequisites Check Attempt $attempt/$MAX_ATTEMPTS ===" - - all_checks_passed=true - - # Download kubeconfigs - if ! download_kubeconfig "$PRIMARY_CLUSTER"; then - echo "Failed to download kubeconfig for $PRIMARY_CLUSTER" - all_checks_passed=false - fi - - if ! download_kubeconfig "$SECONDARY_CLUSTER"; then - echo "Failed to download kubeconfig for $SECONDARY_CLUSTER" - all_checks_passed=false - fi - - if [[ "$all_checks_passed" == "true" ]]; then - # Check Submariner health on individual clusters - if ! check_submariner_health "$PRIMARY_CLUSTER" "$KUBECONFIG_DIR/${PRIMARY_CLUSTER}-kubeconfig.yaml"; then - all_checks_passed=false - fi - - if ! check_submariner_health "$SECONDARY_CLUSTER" "$KUBECONFIG_DIR/${SECONDARY_CLUSTER}-kubeconfig.yaml"; then - all_checks_passed=false - fi - - # Check Submariner connectivity between clusters - if ! check_submariner_connectivity; then - all_checks_passed=false - fi - fi - - if [[ "$all_checks_passed" == "true" ]]; then - echo "๐ŸŽ‰ All Submariner prerequisites are met! Proceeding with DR policy deployment..." - exit 0 - else - echo "โŒ Some Submariner prerequisites are not met. Waiting $SLEEP_INTERVAL seconds before retry..." - progress_sleep "$SLEEP_INTERVAL" - ((attempt++)) - fi - done - - echo "โŒ Submariner prerequisites check failed after $MAX_ATTEMPTS attempts" - echo "๐Ÿ”„ Continuing to retry until all prerequisites are met..." - echo "Please ensure:" - echo "1. Submariner is installed and connected between $PRIMARY_CLUSTER and $SECONDARY_CLUSTER" - echo "2. Submariner operator is running on both managed clusters" - echo "3. Submariner gateway nodes are configured on both managed clusters" - echo "4. Both clusters are registered in the Submariner broker" - echo "" - echo "๐Ÿ”„ Restarting Submariner prerequisites check..." - # Reset attempt counter and continue - attempt=1 - progress_sleep "$SLEEP_INTERVAL" -done # End of infinite retry loop diff --git a/ansible/roles/submariner_prerequisites/tasks/main.yml b/ansible/roles/submariner_prerequisites/tasks/main.yml deleted file mode 100644 index 4758e71..0000000 --- a/ansible/roles/submariner_prerequisites/tasks/main.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Run Submariner prerequisites check - ansible.builtin.command: - argv: - - bash - - "{{ role_path }}/files/submariner-prerequisites-check.sh" - environment: - PRIMARY_CLUSTER: "{{ primary_cluster }}" - SECONDARY_CLUSTER: "{{ secondary_cluster }}" - SUBMARINER_BROKER_NAMESPACE: "{{ submariner_broker_namespace }}" - register: submariner_prereq - changed_when: false - failed_when: false - -- name: Promote Submariner prerequisites script failure to Ansible output - ansible.builtin.debug: - msg: |- - Submariner prerequisites script exited with rc={{ submariner_prereq.rc }}. - ========== stderr ========== - {{ submariner_prereq.stderr | default('', true) }} - ========== stdout (full) ========== - {{ submariner_prereq.stdout | default('', true) }} - when: submariner_prereq.rc != 0 - -- name: Fail play when Submariner prerequisites script failed - ansible.builtin.fail: - msg: >- - Submariner prerequisites script exited with rc={{ submariner_prereq.rc }}. - Full stdout/stderr were printed in the previous debug task. - when: submariner_prereq.rc != 0 - -- name: Promote recent Submariner output after long retry run (tail of stdout) - ansible.builtin.debug: - msg: |- - Submariner prerequisites succeeded (rc=0); last 8000 characters of script output: - {{ (submariner_prereq.stdout | default(''))[-8000:] }} - when: - - submariner_prereq.rc == 0 - - (submariner_prereq.stdout | default('') | length) > 6000 diff --git a/ansible/roles/submariner_sg_tag/files/submariner-sg-tag.sh b/ansible/roles/submariner_sg_tag/files/submariner-sg-tag.sh deleted file mode 100755 index 5d9c506..0000000 --- a/ansible/roles/submariner_sg_tag/files/submariner-sg-tag.sh +++ /dev/null @@ -1,557 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "Starting Submariner security group tagging..." - -# Check if AWS CLI is available, install to user-writable location if needed -AWS_CLI_PATH="/tmp/aws-cli" -if ! command -v aws &>/dev/null; then - echo "AWS CLI is not available. Installing to $AWS_CLI_PATH..." - # Try to install AWS CLI v2 to a user-writable location - if command -v curl &>/dev/null && command -v unzip &>/dev/null; then - curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" - unzip -q /tmp/awscliv2.zip -d /tmp - # Install to user-writable location (no sudo needed) - /tmp/aws/install -i "$AWS_CLI_PATH" -b "$AWS_CLI_PATH/bin" - rm -rf /tmp/aws /tmp/awscliv2.zip - # Add to PATH - export PATH="$AWS_CLI_PATH/bin:$PATH" - else - echo "โŒ Cannot install AWS CLI - required tools (curl, unzip) not available" - exit 1 - fi -fi - -# Verify AWS CLI is working -if ! aws --version &>/dev/null; then - echo "โŒ AWS CLI is not working" - exit 1 -fi - -echo "โœ… AWS CLI is available: $(aws --version 2>&1)" - -# Configuration -KUBECONFIG_DIR="/tmp/kubeconfigs" - -# Create kubeconfig directory -mkdir -p "$KUBECONFIG_DIR" - -# Function to download kubeconfig for a cluster (using same method as download-kubeconfigs.sh) -download_kubeconfig() { - local cluster="$1" - local kubeconfig_path="$KUBECONFIG_DIR/${cluster}-kubeconfig.yaml" - - echo "Downloading kubeconfig for $cluster..." >&2 - - # Check if cluster is available (same as download-kubeconfigs.sh) - local cluster_status=$(oc get managedcluster "$cluster" -o jsonpath='{.status.conditions[?(@.type=="ManagedClusterConditionAvailable")].status}' 2>/dev/null || echo "Unknown") - if [[ "$cluster_status" != "True" ]]; then - echo " โš ๏ธ Cluster $cluster is not available (status: $cluster_status), skipping..." >&2 - return 1 - fi - - # Get the kubeconfig secret name (same method as download-kubeconfigs.sh) - local kubeconfig_secret=$(oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1) - - if [[ -z "$kubeconfig_secret" ]]; then - echo " โŒ No kubeconfig secret found for cluster $cluster" >&2 - return 1 - fi - - echo " Found kubeconfig secret: $kubeconfig_secret" >&2 - - # Try to get the kubeconfig data (same method as download-kubeconfigs.sh) - local kubeconfig_data="" - - # First try to get the 'kubeconfig' field - kubeconfig_data=$(oc get "$kubeconfig_secret" -n "$cluster" -o jsonpath='{.data.kubeconfig}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - - # If that fails, try the 'raw-kubeconfig' field - if [[ -z "$kubeconfig_data" ]]; then - kubeconfig_data=$(oc get "$kubeconfig_secret" -n "$cluster" -o jsonpath='{.data.raw-kubeconfig}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - fi - - if [[ -z "$kubeconfig_data" ]]; then - echo " โŒ Could not extract kubeconfig data for cluster $cluster" >&2 - return 1 - fi - - # Write the kubeconfig to file - echo "$kubeconfig_data" >"$kubeconfig_path" - - # Verify the kubeconfig is valid (same as download-kubeconfigs.sh) - if oc --kubeconfig="$kubeconfig_path" get nodes &>/dev/null; then - echo " โœ… Successfully downloaded and verified kubeconfig for $cluster" >&2 - # Only output the path to stdout (for capture) - echo "$kubeconfig_path" - return 0 - else - echo " โš ๏ธ Downloaded kubeconfig for $cluster but it may not be valid" >&2 - # Still return the path even if validation fails, as it might work for some operations - echo "$kubeconfig_path" - return 0 - fi -} - -# Function to get infrastructure name from a cluster -# Prefer managed cluster infrastructure.status.infrastructureName: that matches AWS (SG names, etc.). -# Hub ClusterDeployment.spec.clusterMetadata.infraID can be a stale Hive attempt ID in BYOC after a -# failed provision, while the joined cluster reflects the real openshift-install infraID. -get_infrastructure_name() { - local cluster="$1" - local kubeconfig="$2" - - # Send debug messages to stderr so they don't interfere with stdout output - echo " Getting infrastructure name for cluster $cluster..." >&2 - - local infra_name="" - - # Method 1: Managed cluster infrastructure status (authoritative for AWS naming) - if [[ -n "$kubeconfig" && -f "$kubeconfig" ]]; then - if oc --kubeconfig="$kubeconfig" get infrastructure cluster &>/dev/null; then - infra_name=$(oc --kubeconfig="$kubeconfig" get infrastructure cluster -o jsonpath='{.status.infrastructureName}' 2>/dev/null || echo "") - if [[ -n "$infra_name" ]]; then - echo " โœ… Infrastructure name from managed cluster infrastructure.status: $infra_name" >&2 - echo "$infra_name" - return 0 - fi - echo " Managed cluster infrastructure.status.infrastructureName empty, trying hub ClusterDeployment..." >&2 - else - echo " No infrastructure.cluster on managed cluster (or no access), trying hub ClusterDeployment..." >&2 - fi - fi - - # Method 2: ClusterDeployment on hub (Hive; may lag or reflect a failed attempt in BYOC) - infra_name=$(oc get clusterdeployment "$cluster" -n "$cluster" -o jsonpath='{.status.infrastructureName}' 2>/dev/null || echo "") - if [[ -z "$infra_name" ]]; then - infra_name=$(oc get clusterdeployment "$cluster" -n "$cluster" -o jsonpath='{.spec.clusterMetadata.infraID}' 2>/dev/null || echo "") - fi - - # Method 3: Managed cluster infrastructure metadata name - if [[ -z "$infra_name" && -n "$kubeconfig" && -f "$kubeconfig" ]]; then - echo " Trying managed cluster infrastructure.metadata.name..." >&2 - infra_name=$(oc --kubeconfig="$kubeconfig" get infrastructure cluster -o jsonpath='{.metadata.name}' 2>/dev/null || echo "") - fi - - # Method 4: Try to extract from install-config secret - if [[ -z "$infra_name" ]]; then - echo " Trying to get infrastructure name from install-config..." >&2 - local install_config_secret="${cluster}-cluster-install-config" - if oc get secret "$install_config_secret" -n "$cluster" &>/dev/null; then - # The infrastructure name is typically the cluster name with a random suffix - # Try to get it from the metadata name in install-config - local cluster_name_from_config=$(oc get secret "$install_config_secret" -n "$cluster" -o jsonpath='{.data.install-config\.yaml}' 2>/dev/null | - base64 -d 2>/dev/null | grep -E '^\s*metadata:' -A 5 | grep -E '^\s*name:' | awk '{print $2}' | tr -d '"' || echo "") - if [[ -n "$cluster_name_from_config" ]]; then - # Infrastructure name is usually cluster name with a random suffix - # We can't get the exact suffix, but we can use the cluster name as a fallback - infra_name="$cluster_name_from_config" - fi - fi - fi - - # Method 5: Use cluster name as fallback (last resort) - if [[ -z "$infra_name" ]]; then - echo " โš ๏ธ Using cluster name as infrastructure name fallback..." >&2 - infra_name="$cluster" - fi - - if [[ -z "$infra_name" ]]; then - echo " โŒ Could not get infrastructure name for cluster $cluster using any method" >&2 - return 1 - fi - - echo " โœ… Infrastructure name: $infra_name" >&2 - # Only output the infra_name to stdout (for capture) - echo "$infra_name" - return 0 -} - -# Function to get AWS credentials from hub cluster -get_aws_credentials() { - local cluster="$1" - local kubeconfig="$2" - - echo " Getting AWS credentials for cluster $cluster..." - - # Try to get AWS credentials from the cluster's AWS creds secret in the hub cluster - local aws_secret_name="${cluster}-cluster-aws-creds" - local aws_access_key="" - local aws_secret_key="" - local aws_region="" - - # Get AWS access key from hub cluster (secret is in the cluster's namespace on hub) - aws_access_key=$(oc get secret "$aws_secret_name" -n "$cluster" -o jsonpath='{.data.aws_access_key_id}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - - if [[ -z "$aws_access_key" ]]; then - echo " โŒ Could not get AWS access key from secret $aws_secret_name in namespace $cluster" - return 1 - fi - - # Get AWS secret key from hub cluster - aws_secret_key=$(oc get secret "$aws_secret_name" -n "$cluster" -o jsonpath='{.data.aws_secret_access_key}' 2>/dev/null | base64 -d 2>/dev/null || echo "") - - if [[ -z "$aws_secret_key" ]]; then - echo " โŒ Could not get AWS secret key from secret $aws_secret_name in namespace $cluster" - return 1 - fi - - # Try multiple methods to get AWS region - aws_region="" - - # Method 1: Try to get from ClusterDeployment on hub cluster - echo " Trying to get region from ClusterDeployment..." - aws_region=$(oc get clusterdeployment "$cluster" -n "$cluster" -o jsonpath='{.spec.platform.aws.region}' 2>/dev/null || echo "") - - # Method 2: Try to get from install-config secret on hub cluster - if [[ -z "$aws_region" ]]; then - echo " Trying to get region from install-config secret..." - local install_config_secret="${cluster}-cluster-install-config" - if oc get secret "$install_config_secret" -n "$cluster" &>/dev/null; then - aws_region=$(oc get secret "$install_config_secret" -n "$cluster" -o jsonpath='{.data.install-config\.yaml}' 2>/dev/null | - base64 -d 2>/dev/null | grep -E '^\s*region:' | awk '{print $2}' | tr -d '"' || echo "") - fi - fi - - # Method 3: Try to get from managed cluster's infrastructure (using managed cluster kubeconfig) - if [[ -z "$aws_region" ]]; then - echo " Trying to get region from managed cluster infrastructure..." - aws_region=$(oc --kubeconfig="$kubeconfig" get infrastructure cluster -o jsonpath='{.status.platformStatus.aws.region}' 2>/dev/null || echo "") - fi - - # Method 4: Try to get from ManagedClusterInfo on hub - if [[ -z "$aws_region" ]]; then - echo " Trying to get region from ManagedClusterInfo..." - aws_region=$(oc get managedclusterinfo "$cluster" -n "$cluster" -o jsonpath='{.status.clusterClaims[?(@.name=="region.open-cluster-management.io")].value}' 2>/dev/null || echo "") - fi - - # Method 5: Try to get from infrastructure spec (fallback) - if [[ -z "$aws_region" ]]; then - echo " Trying to get region from infrastructure spec..." - aws_region=$(oc --kubeconfig="$kubeconfig" get infrastructure cluster -o jsonpath='{.spec.platformSpec.aws.region}' 2>/dev/null || echo "") - fi - - # Method 6: Try to detect from AWS using credentials (if we have them and have infra_name) - # This requires the infrastructure name to search for cluster resources - if [[ -z "$aws_region" && -n "$aws_access_key" && -n "$aws_secret_key" ]]; then - echo " Trying to detect region from AWS resources..." - # Get infrastructure name first (we'll need it to find cluster resources) - local temp_infra_name=$(oc --kubeconfig="$kubeconfig" get infrastructure cluster -o jsonpath='{.status.infrastructureName}' 2>/dev/null || echo "") - - if [[ -n "$temp_infra_name" ]]; then - # Temporarily set credentials - export AWS_ACCESS_KEY_ID="$aws_access_key" - export AWS_SECRET_ACCESS_KEY="$aws_secret_key" - - # Get list of all available regions - local available_regions=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text 2>/dev/null || echo "") - - if [[ -n "$available_regions" ]]; then - # Search for VPCs or security groups tagged with the cluster infrastructure name - for test_region in $available_regions; do - export AWS_DEFAULT_REGION="$test_region" - # Look for VPCs with tags matching the cluster infrastructure name - local vpc_found=$(aws ec2 describe-vpcs \ - --filters "Name=tag:Name,Values=${temp_infra_name}*" \ - --query 'Vpcs[0].VpcId' \ - --output text 2>/dev/null || echo "") - - if [[ -n "$vpc_found" && "$vpc_found" != "None" ]]; then - aws_region="$test_region" - echo " Found cluster VPC in region: $test_region" - break - fi - done - fi - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION - fi - fi - - if [[ -z "$aws_region" ]]; then - echo " โŒ Could not determine AWS region using any method" - echo " Tried: ClusterDeployment, install-config secret, infrastructure status, ManagedClusterInfo, infrastructure spec, AWS detection" - return 1 - fi - - echo " โœ… Successfully determined AWS region: $aws_region" - - # Export AWS credentials - export AWS_ACCESS_KEY_ID="$aws_access_key" - export AWS_SECRET_ACCESS_KEY="$aws_secret_key" - export AWS_DEFAULT_REGION="$aws_region" - - return 0 -} - -# Function to find Submariner security group -find_submariner_security_group() { - local cluster="$1" - local infra_name="$2" - - # Send debug messages to stderr so they don't interfere with stdout output - echo " Finding Submariner security group for cluster $cluster..." >&2 - - # Submariner security groups are typically tagged with submariner-related tags - # Look for security groups with submariner tags or names - local sg_id="" - - # Method 1: Look for security groups tagged with submariner.io/gateway - sg_id=$(aws ec2 describe-security-groups \ - --filters "Name=tag:submariner.io/gateway,Values=true" \ - "Name=tag:Name,Values=*submariner*" \ - --query 'SecurityGroups[0].GroupId' \ - --output text 2>/dev/null || echo "") - - # Method 2: Look for security groups with submariner in the name - if [[ -z "$sg_id" || "$sg_id" == "None" ]]; then - sg_id=$(aws ec2 describe-security-groups \ - --filters "Name=tag:Name,Values=*submariner*" \ - --query 'SecurityGroups[0].GroupId' \ - --output text 2>/dev/null || echo "") - fi - - # Method 3: Look for security groups tagged with the cluster infrastructure name and submariner - if [[ -z "$sg_id" || "$sg_id" == "None" ]]; then - sg_id=$(aws ec2 describe-security-groups \ - --filters "Name=tag:Name,Values=${infra_name}*submariner*" \ - --query 'SecurityGroups[0].GroupId' \ - --output text 2>/dev/null || echo "") - fi - - # Method 4: Look for security groups that are part of the cluster's VPC and have submariner-related names - if [[ -z "$sg_id" || "$sg_id" == "None" ]]; then - # Get VPC ID from cluster infrastructure - local vpc_id=$(oc --kubeconfig="$KUBECONFIG_DIR/${cluster}-kubeconfig.yaml" get infrastructure cluster -o jsonpath='{.status.platformStatus.aws.vpc}' 2>/dev/null || echo "") - - if [[ -n "$vpc_id" && "$vpc_id" != "None" ]]; then - sg_id=$(aws ec2 describe-security-groups \ - --filters "Name=vpc-id,Values=$vpc_id" \ - "Name=group-name,Values=*submariner*" \ - --query 'SecurityGroups[0].GroupId' \ - --output text 2>/dev/null || echo "") - fi - fi - - if [[ -z "$sg_id" || "$sg_id" == "None" ]]; then - echo " โŒ Could not find Submariner security group for cluster $cluster" >&2 - return 1 - fi - - echo " โœ… Found Submariner security group: $sg_id" >&2 - # Only output the sg_id to stdout (for capture) - echo "$sg_id" - return 0 -} - -# Function to tag security group -tag_security_group() { - local cluster="$1" - local infra_name="$2" - local sg_id="$3" - - # Validate inputs - if [[ -z "$infra_name" ]]; then - echo " โŒ Infrastructure name is empty, cannot create tag" - return 1 - fi - - if [[ -z "$sg_id" || "$sg_id" == "None" ]]; then - echo " โŒ Security group ID is empty or invalid, cannot create tag" - return 1 - fi - - # Construct the tag key and value - local tag_key="kubernetes.io/cluster/${infra_name}" - local tag_value="owned" - - echo " Tagging security group $sg_id" - echo " Tag Key: $tag_key" - echo " Tag Value: $tag_value" - - # Check if tag already exists - local existing_tag="" - local check_error="" - existing_tag=$(aws ec2 describe-tags \ - --filters "Name=resource-id,Values=$sg_id" \ - "Name=key,Values=$tag_key" \ - --query 'Tags[0].Value' \ - --output text 2>&1) - check_error=$? - - if [[ $check_error -eq 0 && "$existing_tag" == "$tag_value" ]]; then - echo " โœ… Tag already exists with correct value: $tag_key=$tag_value" - return 0 - fi - - # Create or update the tag - echo " Creating tag..." - local tag_output="" - local tag_error="" - tag_output=$(aws ec2 create-tags \ - --resources "$sg_id" \ - --tags "Key=$tag_key,Value=$tag_value" 2>&1) - tag_error=$? - - if [[ $tag_error -eq 0 ]]; then - echo " โœ… Successfully tagged security group $sg_id with $tag_key=$tag_value" - return 0 - else - echo " โŒ Failed to tag security group $sg_id" - echo " AWS CLI Error Output:" - echo " $tag_output" | sed 's/^/ /' - echo " AWS CLI Exit Code: $tag_error" - return 1 - fi -} - -# Main execution -echo "" -echo "Discovering managed clusters (excluding local-cluster)..." - -# Get all managed clusters (same method as download-kubeconfigs.sh) -ALL_CLUSTERS=$(oc get managedclusters -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - -if [[ -z "$ALL_CLUSTERS" ]]; then - echo "โŒ No managed clusters found" - exit 1 -fi - -# Filter out local-cluster -MANAGED_CLUSTERS="" -for cluster in $ALL_CLUSTERS; do - if [[ "$cluster" != "local-cluster" ]]; then - if [[ -z "$MANAGED_CLUSTERS" ]]; then - MANAGED_CLUSTERS="$cluster" - else - MANAGED_CLUSTERS="$MANAGED_CLUSTERS $cluster" - fi - fi -done - -if [[ -z "$MANAGED_CLUSTERS" ]]; then - echo "โŒ No managed clusters found (excluding local-cluster)" - exit 1 -fi - -echo "Found managed clusters: $MANAGED_CLUSTERS" -echo "" - -SUCCESS_COUNT=0 -FAILED_CLUSTERS=() - -# Process each managed cluster -CLUSTER_NUM=0 -TOTAL_CLUSTERS=$(echo "$MANAGED_CLUSTERS" | wc -w | tr -d '[:space:]') -echo "Total clusters to process: $TOTAL_CLUSTERS" -echo "" - -for cluster in $MANAGED_CLUSTERS; do - CLUSTER_NUM=$((CLUSTER_NUM + 1)) - echo "==========================================" - echo "Processing cluster $CLUSTER_NUM of $TOTAL_CLUSTERS: $cluster" - echo "==========================================" - echo "DEBUG: Starting iteration for cluster: $cluster" - - # Download kubeconfig (using same method as download-kubeconfigs.sh) - # Note: download_kubeconfig sends debug messages to stderr, so only the path is captured to stdout - set +e # Temporarily disable exit on error for kubeconfig download - kubeconfig=$(download_kubeconfig "$cluster") - download_exit=$? - set -e # Re-enable exit on error - - if [[ $download_exit -ne 0 || -z "$kubeconfig" ]]; then - echo "โŒ Failed to download kubeconfig for $cluster, skipping..." - FAILED_CLUSTERS+=("$cluster") - continue - fi - - # Clean up kubeconfig path (remove any trailing whitespace) - kubeconfig=$(echo "$kubeconfig" | tr -d '[:space:]') - - if [[ ! -f "$kubeconfig" ]]; then - echo "โŒ Kubeconfig file does not exist: $kubeconfig, skipping..." - FAILED_CLUSTERS+=("$cluster") - continue - fi - - # Verify kubeconfig works before trying to get infrastructure name - echo " Verifying kubeconfig connection to $cluster..." - if ! oc --kubeconfig="$kubeconfig" get nodes &>/dev/null; then - echo " โš ๏ธ Warning: Cannot connect to cluster $cluster with kubeconfig, but continuing..." - else - echo " โœ… Successfully connected to cluster $cluster" - fi - - # Get infrastructure name - infra_name=$(get_infrastructure_name "$cluster" "$kubeconfig" || echo "") - if [[ -z "$infra_name" ]]; then - echo "โŒ Failed to get infrastructure name for $cluster, skipping..." - echo " Debug: Attempted to get infrastructure name using multiple methods" - FAILED_CLUSTERS+=("$cluster") - continue - fi - - echo " โœ… Retrieved infrastructure name: $infra_name" - - # Get AWS credentials (need kubeconfig for region detection) - # Unset any existing AWS credentials first to avoid conflicts - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION - - set +e # Temporarily disable exit on error for AWS credentials - if ! get_aws_credentials "$cluster" "$kubeconfig"; then - echo "โŒ Failed to get AWS credentials for $cluster, skipping..." - FAILED_CLUSTERS+=("$cluster") - set -e - continue - fi - set -e # Re-enable exit on error - - # Find Submariner security group - sg_id=$(find_submariner_security_group "$cluster" "$infra_name" || echo "") - if [[ -z "$sg_id" ]]; then - echo "โŒ Failed to find Submariner security group for $cluster, skipping..." - FAILED_CLUSTERS+=("$cluster") - continue - fi - - # Tag security group - set +e # Temporarily disable exit on error for tagging - tag_result=0 - if tag_security_group "$cluster" "$infra_name" "$sg_id"; then - echo "โœ… Successfully processed cluster $cluster" - set +e # Disable exit on error for arithmetic - SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) - set -e - tag_result=0 - else - echo "โŒ Failed to tag security group for $cluster" - FAILED_CLUSTERS+=("$cluster") - tag_result=1 - fi - set -e # Re-enable exit on error - - echo "" - echo "Completed processing cluster $cluster (result: $tag_result, success count: $SUCCESS_COUNT)" - echo "DEBUG: Finished iteration for cluster: $cluster, continuing to next cluster..." - echo "" -done - -echo "DEBUG: Exited the for loop. Processed $CLUSTER_NUM clusters." - -echo "==========================================" -echo "Finished processing all clusters. Loop completed." -echo "==========================================" - -# Summary -echo "==========================================" -echo "Summary" -echo "==========================================" -echo "Successfully processed: $SUCCESS_COUNT cluster(s)" -if [[ ${#FAILED_CLUSTERS[@]} -gt 0 ]]; then - echo "Failed clusters: ${FAILED_CLUSTERS[*]}" - exit 1 -else - echo "โœ… All clusters processed successfully" - exit 0 -fi diff --git a/ansible/roles/submariner_sg_tag/tasks/main.yml b/ansible/roles/submariner_sg_tag/tasks/main.yml deleted file mode 100644 index 871abe4..0000000 --- a/ansible/roles/submariner_sg_tag/tasks/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: Tag Submariner security groups on AWS - ansible.builtin.command: - argv: - - bash - - "{{ role_path }}/files/submariner-sg-tag.sh" - register: submariner_sg_tag - changed_when: false - failed_when: false - -- name: Promote Submariner SG tag script failure to Ansible output - ansible.builtin.debug: - msg: |- - submariner-sg-tag.sh exited with rc={{ submariner_sg_tag.rc }}. - ========== stderr ========== - {{ submariner_sg_tag.stderr | default('', true) }} - ========== stdout (full) ========== - {{ submariner_sg_tag.stdout | default('', true) }} - when: submariner_sg_tag.rc != 0 - -- name: Fail play when Submariner SG tagging failed - ansible.builtin.fail: - msg: >- - Submariner SG tag script exited with rc={{ submariner_sg_tag.rc }}. - Full stdout/stderr were printed in the previous debug task. - when: submariner_sg_tag.rc != 0 diff --git a/scripts/ansible/odf-ssl-certificate-extraction.yml b/scripts/ansible/odf-ssl-certificate-extraction.yml deleted file mode 100644 index 11b60cd..0000000 --- a/scripts/ansible/odf-ssl-certificate-extraction.yml +++ /dev/null @@ -1,404 +0,0 @@ ---- -# ODF SSL certificate extraction and distribution. -# Requires: kubernetes.core. Env: PRIMARY_CLUSTER, SECONDARY_CLUSTER. Optional KUBECONFIG when not in-cluster. -# Do not set empty KUBECONFIG on the play โ€” it breaks hub API access in the Job pod (use SA token). -# Run: ansible-playbook -i localhost, -c local odf-ssl-certificate-extraction.yml -- name: ODF SSL certificate extraction - hosts: localhost - gather_facts: true - vars: - work_dir: "/tmp/odf-ssl-certs" - primary_cluster: "{{ lookup('env', 'PRIMARY_CLUSTER') | default('ocp-primary', true) }}" - secondary_cluster: "{{ lookup('env', 'SECONDARY_CLUSTER') | default('ocp-secondary', true) }}" - required_clusters: ["hub", "{{ primary_cluster }}", "{{ secondary_cluster }}"] - kubeconfig: "{{ lookup('env', 'KUBECONFIG') | default('', true) | trim }}" - cluster_readiness_max_attempts: "{{ lookup('env', 'CLUSTER_READINESS_MAX_ATTEMPTS') | default(150, true) | int }}" - cluster_readiness_sleep: "{{ lookup('env', 'CLUSTER_READINESS_SLEEP_SECONDS') | default(30, true) | int }}" - tasks: - - name: Log job context (cluster names, paths, auth hint) - ansible.builtin.debug: - msg: - - "ODF SSL extraction starting" - - "work_dir={{ work_dir }}" - - "primary_cluster={{ primary_cluster }} secondary_cluster={{ secondary_cluster }}" - - "KUBECONFIG_from_env={{ 'set' if (kubeconfig | length) > 0 else 'unset (in-cluster SA token expected)' }}" - - "managed_cluster_kubeconfigs_expected_under={{ work_dir }}/-kubeconfig.yaml" - - - name: Create work directory - ansible.builtin.file: - path: "{{ work_dir }}" - state: directory - mode: "0755" - - - name: Wait for both managed clusters to be up before extraction - ansible.builtin.include_tasks: tasks/wait-dr-managedclusters-available.yml - - - name: Extract hub CA (current context) - ansible.builtin.include_tasks: tasks/odf-ssl-extract-ca.yml - vars: - cluster_name: hub - output_path: "{{ work_dir }}/hub-ca.crt" - kubeconfig_path: "" - - - name: Get ingress CA from hub (router-ca secret) - block: - - name: Get hub ingress CA secret - kubernetes.core.k8s_info: - kind: Secret - name: router-ca - namespace: openshift-ingress-operator - register: router_ca - - name: Write hub ingress CA certificate - ansible.builtin.copy: - content: "{{ (router_ca.resources[0].data.get('tls.crt') or router_ca.resources[0].data.get('ca.crt', '')) | b64decode }}" - dest: "{{ work_dir }}/hub-ingress-ca.crt" - mode: "0644" - when: router_ca.resources | length > 0 and (router_ca.resources[0].data | length > 0) - rescue: - - name: Report missing hub ingress CA secret - ansible.builtin.debug: - msg: "Hub ingress CA not available, continuing without it" - - - name: Get managed clusters - kubernetes.core.k8s_info: - api_version: cluster.open-cluster-management.io/v1 - kind: ManagedCluster - register: managed_clusters_result - - - name: Set managed clusters list (exclude local-cluster) - ansible.builtin.set_fact: - _mc_names: >- - {{ - managed_clusters_result.resources | default([]) - | map(attribute='metadata.name') - | reject('equalto', 'local-cluster') - | list - }} - - - name: Require PRIMARY_CLUSTER and SECONDARY_CLUSTER to match hub ManagedCluster names - ansible.builtin.fail: - msg: >- - PRIMARY_CLUSTER={{ primary_cluster }} and SECONDARY_CLUSTER={{ secondary_cluster }} must each match an ACM - ManagedCluster on this hub (excl. local-cluster). Discovered: {{ _mc_names | join(', ') | default('none', true) }}. - Set Helm regionalDR (or env) so names match metadata.name โ€” do not use placeholders like ocp-primary if clusters are named differently. - when: >- - (primary_cluster not in _mc_names) - or (secondary_cluster not in _mc_names) - - - name: Set managed clusters list (hub discovery only โ€” avoids bogus kubeconfig namespaces) - ansible.builtin.set_fact: - managed_clusters: "{{ _mc_names | sort }}" - - - name: Download kubeconfig for managed cluster - ansible.builtin.include_tasks: tasks/kubeconfig-attempt.yml - loop: "{{ managed_clusters }}" - loop_control: - label: "{{ item }}" - vars: - cluster: "{{ item }}" - kubeconfig_path: "{{ work_dir }}/{{ item }}-kubeconfig.yaml" - check_available: false - - - name: Fail if kubeconfig could not be written for PRIMARY or SECONDARY - ansible.builtin.fail: - msg: >- - Missing {{ work_dir }}/{{ item }}-kubeconfig.yaml (needed for CA extract and cluster-proxy distribution). - kubeconfig_failures={{ kubeconfig_failures | default([]) }}. - Fix hub RBAC, cluster import, or secret name (admin-kubeconfig / kubeconfig) in namespace {{ item }}. - loop: "{{ [primary_cluster, secondary_cluster] }}" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is not file - - - name: Extract CA from managed cluster - ansible.builtin.include_tasks: tasks/odf-ssl-extract-ca.yml - loop: "{{ managed_clusters }}" - loop_control: - label: "{{ item }}" - vars: - cluster_name: "{{ item }}" - output_path: "{{ work_dir }}/{{ item }}-ca.crt" - kubeconfig_path: "{{ work_dir }}/{{ item }}-kubeconfig.yaml" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is file - - - name: Get ingress CA from managed cluster - block: - - name: Get managed cluster ingress CA secret - kubernetes.core.k8s_info: - kind: Secret - name: router-ca - namespace: openshift-ingress-operator - kubeconfig: "{{ work_dir }}/{{ item }}-kubeconfig.yaml" - register: router_ca_mc - loop: "{{ managed_clusters }}" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is file - loop_control: - label: "{{ item }}" - failed_when: false - - name: Write managed cluster ingress CA certificate - ansible.builtin.copy: - content: "{{ (item.resources[0].data.get('tls.crt') or item.resources[0].data.get('ca.crt', '')) | b64decode }}" - dest: "{{ work_dir }}/{{ item.item }}-ingress-ca.crt" - mode: "0644" - loop: "{{ router_ca_mc.results | default([]) }}" - when: item.resources | length > 0 and (item.resources[0].data | length > 0) - failed_when: false - - name: Find CA files in work dir - ansible.builtin.find: - paths: "{{ work_dir }}" - patterns: "*.crt" - register: ca_files_found - - # Each path must be a separate shell word; join+quote made one bogus path and produced an empty bundle. - - name: Create combined CA bundle (first 5 certs per file) - ansible.builtin.shell: | - set -e - out="{{ work_dir }}/combined-ca-bundle.crt" - : > "$out" - for f in \ - {% for p in ca_files_found.files %} - {% if loop.last %} - {{ p.path | quote }} - {% else %} - {{ p.path | quote }} \ - {% endif %} - {% endfor %} - do - [ -s "$f" ] || continue - echo "# CA from $(basename "$f" .crt)" >> "$out" - cert_count=0 - in_cert=false - while IFS= read -r line; do - if [ "$line" = "-----BEGIN CERTIFICATE-----" ]; then in_cert=true; cert_count=$((cert_count+1)); [ $cert_count -le 5 ] || break; fi - [ "$in_cert" = true ] && echo "$line" >> "$out" - if [ "$line" = "-----END CERTIFICATE-----" ]; then in_cert=false; echo "" >> "$out"; fi - done < "$f" - done - args: - executable: /bin/bash - when: ca_files_found.matched | default(0) > 0 - changed_when: false - - - name: Stat combined CA bundle - ansible.builtin.stat: - path: "{{ work_dir }}/combined-ca-bundle.crt" - register: combined_ca_stat - when: ca_files_found.matched | default(0) > 0 - - - name: Fail when combined CA bundle is empty or missing - ansible.builtin.fail: - msg: >- - combined-ca-bundle.crt is missing or too small ({{ combined_ca_stat.stat.size | default(0) }} bytes). - CA .crt files were found but the merge script produced no output โ€” check paths and permissions. - when: >- - ca_files_found.matched | default(0) | int > 0 - and (combined_ca_stat.stat.size | default(0) | int) < 64 - - - name: Validate required clusters have CA - ansible.builtin.fail: - msg: >- - CA material: need PEM files under {{ work_dir }} from hub + PRIMARY + SECONDARY (Helm regionalDR names must match ACM ManagedCluster names). - hub-ca.crt={{ 'present' if (work_dir + '/hub-ca.crt') is file else 'MISSING' }}, - {{ primary_cluster }}-ca.crt={{ 'present' if (work_dir + '/' + primary_cluster + '-ca.crt') is file else 'MISSING' }}, - {{ secondary_cluster }}-ca.crt={{ 'present' if (work_dir + '/' + secondary_cluster + '-ca.crt') is file else 'MISSING' }}. - If a spoke file is MISSING: check kubeconfig was saved as {{ work_dir }}/-kubeconfig.yaml, - and that ConfigMap trusted-ca-bundle exists in openshift-config-managed on that cluster (kubernetes.core k8s_info errors appear earlier). - when: (work_dir + '/hub-ca.crt') is not file or (work_dir + '/' + primary_cluster + '-ca.crt') is not file or (work_dir + '/' + secondary_cluster + - '-ca.crt') is not file - - - name: Get cluster-proxy-ca-bundle for merge - kubernetes.core.k8s_info: - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - register: cluster_proxy_cm_get - failed_when: false - - - name: Create cluster-proxy-ca-bundle ConfigMap - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: cluster-proxy-ca-bundle - namespace: openshift-config - data: - ca-bundle.crt: "{{ lookup('file', work_dir + '/combined-ca-bundle.crt') }}" - when: cluster_proxy_cm_get.resources | length == 0 - - - name: Patch cluster-proxy-ca-bundle ConfigMap - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: cluster-proxy-ca-bundle - namespace: openshift-config - data: - ca-bundle.crt: "{{ lookup('file', work_dir + '/combined-ca-bundle.crt') }}" - when: cluster_proxy_cm_get.resources | length > 0 - - - name: Update hub cluster proxy trustedCA - kubernetes.core.k8s: - state: present - definition: - apiVersion: config.openshift.io/v1 - kind: Proxy - metadata: - name: cluster - spec: - trustedCA: - name: cluster-proxy-ca-bundle - failed_when: false - - # Push to spokes here (before ramenddr restarts / Velero) so later steps cannot block distribution. - # Use oc --from-file | apply like odf-ssl-certificate-extraction.sh โ€” avoids kubernetes.core + huge inline data. - - name: Stat combined bundle before spoke push - ansible.builtin.command: stat -c '%s' "{{ work_dir }}/combined-ca-bundle.crt" - register: odf_ssl_bundle_bytes - changed_when: false - - - name: Log spoke distribution prerequisites - ansible.builtin.debug: - msg: - - "Pushing cluster-proxy-ca-bundle to each managed cluster (oc); combined-ca-bundle.crt bytes={{ odf_ssl_bundle_bytes.stdout | default('n/a') }}" - - "managed_clusters={{ managed_clusters }} kubeconfig files under {{ work_dir }}/*-kubeconfig.yaml" - - - name: Distribute cluster-proxy-ca-bundle to managed cluster via oc (from-file + apply, 3 retries) - ansible.builtin.shell: | - set -euo pipefail - set -o pipefail - K="{{ work_dir }}/{{ item }}-kubeconfig.yaml" - B="{{ work_dir }}/combined-ca-bundle.crt" - echo "=== odf-ssl: cluster-proxy-ca-bundle -> {{ item }} (kubeconfig=$K) ===" - test -s "$K" - test -s "$B" - for attempt in 1 2 3; do - echo " attempt ${attempt}/3 ..." - if oc --kubeconfig="$K" create configmap cluster-proxy-ca-bundle \ - --from-file=ca-bundle.crt="$B" \ - -n openshift-config \ - --dry-run=client -o yaml \ - | oc --kubeconfig="$K" apply -f -; then - if oc --kubeconfig="$K" patch proxy/cluster --type=merge \ - --patch='{"spec":{"trustedCA":{"name":"cluster-proxy-ca-bundle"}}}'; then - echo "RESULT ok cluster={{ item }}" - exit 0 - else - echo " proxy patch failed (see oc stderr above)" >&2 - fi - else - echo " configmap apply failed (see oc stderr above)" >&2 - fi - if [[ "$attempt" -lt 3 ]]; then sleep 10; fi - done - echo "RESULT fail cluster={{ item }} after 3 attempts" >&2 - exit 1 - args: - executable: /bin/bash - loop: "{{ managed_clusters }}" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is file - loop_control: - label: "{{ item }}" - register: mc_oc_distribute - failed_when: false - - - name: Fail when oc distribution failed on a spoke (full stdout/stderr) - ansible.builtin.fail: - msg: |- - cluster-proxy-ca-bundle was NOT applied on {{ item.item }}. - Common causes: Forbidden (kubeconfig user cannot create ConfigMap in openshift-config or patch proxies/cluster), - invalid/expired kubeconfig, or network to the spoke API. - --- stdout --- - {{ item.stdout | default('', true) }} - --- stderr --- - {{ item.stderr | default('', true) }} - loop: "{{ mc_oc_distribute.results | default([]) }}" - when: not item.skipped | default(false) and (item.rc | default(1)) != 0 - - - name: Verify cluster-proxy-ca-bundle and proxy.trustedCA on each spoke (oc) - ansible.builtin.shell: | - set -euo pipefail - K="{{ work_dir }}/{{ item }}-kubeconfig.yaml" - sz=$(oc --kubeconfig="$K" get configmap cluster-proxy-ca-bundle -n openshift-config \ - -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c | tr -d ' ' || echo 0) - trust=$(oc --kubeconfig="$K" get proxy cluster -o jsonpath='{.spec.trustedCA.name}' 2>/dev/null || echo "") - echo "cluster={{ item }} ca-bundle.crt_bytes=$sz spec.trustedCA.name=$trust" - test "${sz:-0}" -ge 64 - test "$trust" = "cluster-proxy-ca-bundle" - args: - executable: /bin/bash - loop: "{{ managed_clusters }}" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is file - loop_control: - label: "{{ item }}" - register: mc_oc_verify - failed_when: false - - - name: Fail when post-push verification failed on a spoke - ansible.builtin.fail: - msg: |- - Verification failed on {{ item.item }} (ConfigMap empty/missing or Proxy.spec.trustedCA not cluster-proxy-ca-bundle). - Re-check ACM policies on the spoke that might overwrite cluster-proxy-ca-bundle. - --- stdout --- - {{ item.stdout | default('', true) }} - --- stderr --- - {{ item.stderr | default('', true) }} - loop: "{{ mc_oc_verify.results | default([]) }}" - when: not item.skipped | default(false) and (item.rc | default(1)) != 0 - - - name: Get ramenddr pods per cluster - kubernetes.core.k8s_info: - kind: Pod - namespace: openshift-dr-system - kubeconfig: "{{ work_dir }}/{{ item }}-kubeconfig.yaml" - label_selectors: - - "app=ramenddr-cluster-operator" - register: ramen_pods_per_cluster - loop: "{{ managed_clusters }}" - loop_control: - label: "{{ item }}" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is file - failed_when: false - - - name: Delete ramenddr-cluster-operator pods on managed clusters - ansible.builtin.include_tasks: tasks/odf-ssl-delete-pods.yml - loop: "{{ ramen_pods_per_cluster.results | default([]) }}" - loop_control: - loop_var: ramen_pod_query - label: "{{ ramen_pod_query.item }}" - when: ramen_pod_query.resources | default([]) | length > 0 - vars: - target_cluster: "{{ ramen_pod_query.item }}" - pods_to_delete: "{{ ramen_pod_query.resources | default([]) }}" - pod_namespace: openshift-dr-system - - - name: Get Velero pods per cluster - kubernetes.core.k8s_info: - kind: Pod - namespace: openshift-adp - kubeconfig: "{{ work_dir }}/{{ item }}-kubeconfig.yaml" - label_selectors: - - "component=velero" - register: velero_pods_per_cluster - loop: "{{ managed_clusters }}" - when: (work_dir + '/' + item + '-kubeconfig.yaml') is file - loop_control: - label: "{{ item }}" - failed_when: false - - - name: Delete Velero pods on managed clusters - ansible.builtin.include_tasks: tasks/odf-ssl-delete-pods.yml - loop: "{{ velero_pods_per_cluster.results | default([]) }}" - loop_control: - loop_var: velero_pod_query - label: "{{ velero_pod_query.item }}" - when: velero_pod_query.resources | default([]) | length > 0 - vars: - target_cluster: "{{ velero_pod_query.item }}" - pods_to_delete: "{{ velero_pod_query.resources | default([]) }}" - pod_namespace: openshift-adp - - - name: ODF SSL certificate extraction completed - ansible.builtin.debug: - msg: "ODF SSL certificate management completed successfully." diff --git a/scripts/ansible/odf-ssl-precheck.yml b/scripts/ansible/odf-ssl-precheck.yml deleted file mode 100644 index 77bad25..0000000 --- a/scripts/ansible/odf-ssl-precheck.yml +++ /dev/null @@ -1,250 +0,0 @@ ---- -# ODF SSL certificate precheck. Ensures certificates are distributed before DR policies. -# Can trigger certificate extraction job if distribution is incomplete. -# Requires: kubernetes.core. Env: PRIMARY_CLUSTER, SECONDARY_CLUSTER. -- name: ODF SSL certificate precheck - hosts: localhost - gather_facts: true - vars: - primary_cluster: "{{ lookup('env', 'PRIMARY_CLUSTER') | default('ocp-primary', true) }}" - secondary_cluster: "{{ lookup('env', 'SECONDARY_CLUSTER') | default('ocp-secondary', true) }}" - # Extractor caps at 5 PEMs per source file; not every file has 5. 15 certs was often unreachable. - min_certificates: "{{ lookup('env', 'MIN_CA_CERTIFICATES') | default(6, true) | int }}" - min_bundle_size: "{{ lookup('env', 'MIN_CA_BUNDLE_BYTES') | default(4096, true) | int }}" - max_attempts: 120 - sleep_interval: 30 - cluster_readiness_max_attempts: 150 - cluster_readiness_sleep: 30 - # Precheck is Argo sync wave 2; odf-ssl-certificate-extractor is wave 1 and may still be running. - bundle_wait_retries: "{{ lookup('env', 'BUNDLE_WAIT_RETRIES') | default(60, true) | int }}" - bundle_wait_delay: "{{ lookup('env', 'BUNDLE_WAIT_DELAY_SECONDS') | default(30, true) | int }}" - cluster_ca_mgt_namespace: "{{ lookup('env', 'CLUSTER_CA_MGT_NAMESPACE') | default('cluster-ca-mgt', true) }}" - kubeconfig: "{{ lookup('env', 'KUBECONFIG') | default('', true) | trim }}" - tasks: - # Waits cluster_readiness_sleep seconds BETWEEN k8s_info attempts, same Pod (Ansible until/retries/delay). - # Many distinct Pods usually means OOM/restarts (Job backoffLimit) or Argo PostSync re-creating the hook Jobโ€”not one Pod per retry. - - name: Wait for required ManagedClusters on hub (excl. local-cluster, min 2) - kubernetes.core.k8s_info: - api_version: cluster.open-cluster-management.io/v1 - kind: ManagedCluster - register: mc_wait - until: >- - (mc_wait.resources | default([]) - | map(attribute='metadata.name') - | reject('equalto', 'local-cluster') - | list - | length | int) >= 2 - retries: "{{ cluster_readiness_max_attempts | int }}" - delay: "{{ cluster_readiness_sleep | int }}" - - - name: Set clusters ready after wait - ansible.builtin.set_fact: - clusters_ready: true - - - name: Probe hub cluster-proxy-ca-bundle before wait - kubernetes.core.k8s_info: - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - register: hub_cm_probe - failed_when: false - - - name: Delete empty hub cluster-proxy-ca-bundle (shell blocks extractor create/patch) - kubernetes.core.k8s: - state: absent - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - when: >- - hub_cm_probe.resources | default([]) | length > 0 - and ( - (hub_cm_probe.resources[0].data | default({})).get('ca-bundle.crt', '') | default('', true) | trim | length - ) == 0 - failed_when: false - - - name: Wait for hub cluster-proxy-ca-bundle after extractor Job (wave 1) - kubernetes.core.k8s_info: - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - register: hub_cm_poll - until: >- - hub_cm_poll.resources | default([]) | length > 0 - and ( - (hub_cm_poll.resources[0].data | default({})).get('ca-bundle.crt', '') | default('', true) | length | int - ) >= (min_bundle_size | int) - and ( - (hub_cm_poll.resources[0].data | default({})).get('ca-bundle.crt', '') | default('', true) - | regex_findall('-----BEGIN CERTIFICATE-----') - | length | int - ) >= (min_certificates | int) - retries: "{{ bundle_wait_retries | int }}" - delay: "{{ bundle_wait_delay | int }}" - failed_when: false - - - name: Check certificate distribution on hub - block: - - name: Get cluster-proxy-ca-bundle ConfigMap - kubernetes.core.k8s_info: - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - register: ca_bundle_cm - - - name: Fail when CA bundle ConfigMap not found - ansible.builtin.fail: - msg: "CA bundle ConfigMap not found on hub cluster." - when: ca_bundle_cm.resources | length == 0 - - - name: Set bundle content and size - ansible.builtin.set_fact: - bundle_content: "{{ (ca_bundle_cm.resources[0].data | default({})).get('ca-bundle.crt', '') }}" - bundle_size: "{{ ((ca_bundle_cm.resources[0].data | default({})).get('ca-bundle.crt', '') | length) }}" - when: ca_bundle_cm.resources | length > 0 - - - name: Fail when CA bundle too small - ansible.builtin.fail: - msg: "CA bundle too small ({{ bundle_size | int }} < {{ min_bundle_size | int }} bytes)." - when: bundle_size | default(0) | int < min_bundle_size | int - - - name: Count certificates in bundle - ansible.builtin.set_fact: - cert_count: "{{ bundle_content | default('') | regex_findall('-----BEGIN CERTIFICATE-----') | length | int }}" - - - name: Fail when too few certificates - ansible.builtin.fail: - msg: "Too few certificates in bundle ({{ cert_count }} < {{ min_certificates | int }})." - when: cert_count | default(0) | int < min_certificates | int - - - name: Set certificate distribution complete - ansible.builtin.set_fact: - certificate_distribution_ok: true - rescue: - - name: Set certificate distribution incomplete - ansible.builtin.set_fact: - certificate_distribution_ok: false - - - name: Get managed clusters for cleanup - kubernetes.core.k8s_info: - api_version: cluster.open-cluster-management.io/v1 - kind: ManagedCluster - register: mc_list - when: not (certificate_distribution_ok | default(false) | bool) - - - name: Download kubeconfig for managed cluster (for cleanup) - ansible.builtin.include_tasks: tasks/kubeconfig-attempt.yml - loop: "{{ (mc_list.resources | default([]) | map(attribute='metadata.name') | reject('equalto', 'local-cluster') | list) }}" - loop_control: - label: "{{ item }}" - vars: - cluster: "{{ item }}" - kubeconfig_path: "/tmp/{{ item }}-kubeconfig.yaml" - check_available: false - when: not (certificate_distribution_ok | default(false) | bool) and mc_list.resources is defined - - - name: Cleanup placeholder ConfigMaps on managed clusters - when: not (certificate_distribution_ok | default(false) | bool) - - block: - - name: Get cluster-proxy-ca-bundle on managed cluster - kubernetes.core.k8s_info: - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - kubeconfig: "/tmp/{{ item }}-kubeconfig.yaml" - register: mc_ca_cm - loop: "{{ (mc_list.resources | default([]) | map(attribute='metadata.name') | reject('equalto', 'local-cluster') | list) }}" - failed_when: false - - name: Delete placeholder or empty cluster-proxy-ca-bundle on managed cluster - kubernetes.core.k8s: - state: absent - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - kubeconfig: "/tmp/{{ item.item }}-kubeconfig.yaml" - loop: "{{ mc_ca_cm.results | default([]) }}" - when: >- - item.resources | length > 0 and - ( - (item.resources[0].data.get('ca-bundle.crt', '') | default('', true) | trim | length) == 0 - or "Placeholder for ODF SSL" in (item.resources[0].data.get('ca-bundle.crt', '') or '') - or "This will be populated" in (item.resources[0].data.get('ca-bundle.crt', '') or '') - ) - failed_when: false - - name: Trigger certificate extraction (create Job in workload namespace - handled by ArgoCD/Helm) - ansible.builtin.debug: - msg: >- - Certificate distribution incomplete. Trigger or re-sync the - odf-ssl-certificate-extractor Job in {{ cluster_ca_mgt_namespace }} namespace, - then re-run this precheck. - when: not (certificate_distribution_ok | default(false) | bool) - - - name: Re-check certificate distribution after extraction - when: not (certificate_distribution_ok | default(false) | bool) - - block: - - name: Get cluster-proxy-ca-bundle again - kubernetes.core.k8s_info: - kind: ConfigMap - name: cluster-proxy-ca-bundle - namespace: openshift-config - register: ca_bundle_cm2 - - name: Set certificate_distribution_ok after extraction - ansible.builtin.set_fact: - certificate_distribution_ok: >- - {{ - ( - (ca_bundle_cm2.resources[0].data.get('ca-bundle.crt', '') | default('', true) | length | int) - >= (min_bundle_size | int) - and - ( - ca_bundle_cm2.resources[0].data.get('ca-bundle.crt', '') - | default('', true) - | regex_findall('-----BEGIN CERTIFICATE-----') - | length - | int - ) >= (min_certificates | int) - ) | bool - }} - when: ca_bundle_cm2.resources | length > 0 - - - name: Hub cluster-proxy-ca-bundle missing on re-check - ansible.builtin.debug: - msg: "cluster-proxy-ca-bundle not found on hub openshift-config after extraction wait โ€” distribution still incomplete." - when: ca_bundle_cm2.resources | length == 0 - - - name: Explain hub bundle still below precheck thresholds - ansible.builtin.debug: - msg: >- - Hub cluster-proxy-ca-bundle: {{ ca_bundle_cm2.resources[0].data.get('ca-bundle.crt', '') | default('', true) | length }} bytes, - {{ ca_bundle_cm2.resources[0].data.get('ca-bundle.crt', '') | default('', true) | regex_findall('-----BEGIN CERTIFICATE-----') | length }} PEMs; - need >= {{ min_bundle_size | int }} bytes and >= {{ min_certificates | int }} PEMs (override with MIN_CA_BUNDLE_BYTES / MIN_CA_CERTIFICATES). - If size is 0 but the extractor Job succeeded, check ACM policy is not enforcing a metadata-only - cluster-proxy-ca-bundle on the hub (that reconciles away ca-bundle.crt). - when: >- - ca_bundle_cm2.resources | length > 0 - and not ( - ( - (ca_bundle_cm2.resources[0].data.get('ca-bundle.crt', '') | default('', true) | length | int) >= (min_bundle_size | int) - and - ( - ca_bundle_cm2.resources[0].data.get('ca-bundle.crt', '') - | default('', true) - | regex_findall('-----BEGIN CERTIFICATE-----') - | length | int - ) >= (min_certificates | int) - ) | bool - ) - - name: Fail when certificate distribution still incomplete after extraction - ansible.builtin.fail: - msg: >- - Hub cluster-proxy-ca-bundle still below thresholds (see prior debug). - Thresholds: MIN_CA_BUNDLE_BYTES={{ min_bundle_size | int }}, MIN_CA_CERTIFICATES={{ min_certificates | int }}. - Re-run odf-ssl-certificate-extractor on the hub after fixing any ACM policy that enforces an empty - cluster-proxy-ca-bundle ConfigMap (metadata-only template removes ca-bundle.crt). - Increase BUNDLE_WAIT_RETRIES / BUNDLE_WAIT_DELAY_SECONDS if wave-1 extractor needs more time before this hook. - when: not (certificate_distribution_ok | default(false) | bool) - - - name: Precheck success - ansible.builtin.debug: - msg: "ODF SSL certificate precheck completed successfully. Ready for DR prerequisites check." diff --git a/scripts/ansible/requirements.yml b/scripts/ansible/requirements.yml deleted file mode 100644 index 54f4635..0000000 --- a/scripts/ansible/requirements.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# Install with: ansible-galaxy collection install -r scripts/ansible/requirements.yml -collections: - - name: kubernetes.core - version: ">=4.0.0" diff --git a/scripts/ansible/tasks/kubeconfig-attempt.yml b/scripts/ansible/tasks/kubeconfig-attempt.yml deleted file mode 100644 index ba9c64c..0000000 --- a/scripts/ansible/tasks/kubeconfig-attempt.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Retrieve kubeconfig for {{ cluster }} - block: - - name: Include kubeconfig retrieval tasks for {{ cluster }} - ansible.builtin.include_tasks: tasks/kubeconfig.yml - rescue: - - name: Record kubeconfig retrieval failure for {{ cluster }} - ansible.builtin.set_fact: - kubeconfig_failures: "{{ (kubeconfig_failures | default([])) + [cluster] }}" diff --git a/scripts/ansible/tasks/kubeconfig.yml b/scripts/ansible/tasks/kubeconfig.yml deleted file mode 100644 index ce8a669..0000000 --- a/scripts/ansible/tasks/kubeconfig.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -# Shared tasks: retrieve kubeconfig for a managed cluster (hub context). -# Uses same method as download-kubeconfigs: secret in cluster NS, kubeconfig or raw-kubeconfig. -# Call with: cluster, kubeconfig_path (default /tmp/{{ cluster }}-kubeconfig.yaml) -# Expects: hub kubeconfig in env KUBECONFIG or in-cluster. -- name: Check managed cluster is available - kubernetes.core.k8s_info: - kind: ManagedCluster - name: "{{ cluster }}" - register: mc - failed_when: mc.resources | length == 0 - when: check_available | default(true) | bool - -- name: Get managed cluster availability status - ansible.builtin.set_fact: - cluster_available: >- - {{ - ( - mc.resources[0].status.conditions - | selectattr('type', 'equalto', 'ManagedClusterConditionAvailable') - | map(attribute='status') - | first - | default('') - ) == 'True' - }} - when: check_available | default(true) | bool and mc.resources | length > 0 - -- name: Fail when cluster is not available - ansible.builtin.fail: - msg: "Cluster {{ cluster }} is not available (status not True), skipping." - when: check_available | default(true) | bool and not (cluster_available | default(false)) - -- name: List secrets in managed cluster namespace - kubernetes.core.k8s_info: - kind: Secret - namespace: "{{ cluster }}" - register: secrets_list - -- name: Find kubeconfig secret name (admin-kubeconfig or kubeconfig) - ansible.builtin.set_fact: - kubeconfig_secret_name: >- - {{ - secrets_list.resources - | selectattr('metadata.name', 'match', '.*(admin-kubeconfig|kubeconfig).*') - | map(attribute='metadata.name') - | list - | first - | default('') - }} - when: secrets_list.resources is defined and secrets_list.resources | length > 0 - -- name: Fail when no kubeconfig secret found - ansible.builtin.fail: - msg: "No kubeconfig secret found for cluster {{ cluster }}" - when: kubeconfig_secret_name is not defined or kubeconfig_secret_name == '' - -- name: Get kubeconfig secret - kubernetes.core.k8s_info: - kind: Secret - name: "{{ kubeconfig_secret_name }}" - namespace: "{{ cluster }}" - register: kubeconfig_secret - -- name: Decode kubeconfig from secret (try kubeconfig then raw-kubeconfig) - ansible.builtin.set_fact: - kubeconfig_content: >- - {{ - (kubeconfig_secret.resources[0].data.get('kubeconfig', '') | b64decode) - or (kubeconfig_secret.resources[0].data.get('raw-kubeconfig', '') | b64decode) - or '' - }} - when: kubeconfig_secret.resources | length > 0 - -- name: Fail when kubeconfig data is empty - ansible.builtin.fail: - msg: "Could not extract kubeconfig data for cluster {{ cluster }}" - when: kubeconfig_content is not defined or kubeconfig_content == '' - -- name: Write kubeconfig to file - ansible.builtin.copy: - content: "{{ kubeconfig_content }}" - dest: "{{ kubeconfig_path | default('/tmp/' + cluster + '-kubeconfig.yaml') }}" - mode: "0600" - when: kubeconfig_content is defined and kubeconfig_content != '' diff --git a/scripts/ansible/tasks/odf-ssl-delete-pods.yml b/scripts/ansible/tasks/odf-ssl-delete-pods.yml deleted file mode 100644 index b54fc55..0000000 --- a/scripts/ansible/tasks/odf-ssl-delete-pods.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Delete pods in a list. Vars: target_cluster, pods_to_delete, work_dir, pod_namespace. -- name: Delete listed pod - kubernetes.core.k8s: - state: absent - kind: Pod - namespace: "{{ pod_namespace }}" - name: "{{ del_pod.metadata.name }}" - kubeconfig: "{{ work_dir }}/{{ target_cluster }}-kubeconfig.yaml" - loop: "{{ pods_to_delete }}" - loop_control: - loop_var: del_pod - label: "{{ del_pod.metadata.name }}" - failed_when: false diff --git a/scripts/ansible/tasks/odf-ssl-extract-ca.yml b/scripts/ansible/tasks/odf-ssl-extract-ca.yml deleted file mode 100644 index e9f4829..0000000 --- a/scripts/ansible/tasks/odf-ssl-extract-ca.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -# Extract CA from a cluster. Vars: cluster_name, output_path, kubeconfig_path (omit for hub). -- name: Get trusted-ca-bundle ConfigMap - kubernetes.core.k8s_info: - kind: ConfigMap - name: trusted-ca-bundle - namespace: openshift-config-managed - kubeconfig: >- - {{ - kubeconfig_path - if (kubeconfig_path | default('', true) | trim | length) > 0 - else omit - }} - register: ca_cm - failed_when: false - -- name: Set CA content from ConfigMap data - ansible.builtin.set_fact: - ca_content: >- - {{ - ( - ca_cm.resources[0].data.get('ca-bundle.crt') - or ca_cm.resources[0].data.get('ca-bundle\\.crt', '') - ) if (ca_cm.resources | length > 0 and ca_cm.resources[0].data) else '' - }} - when: ca_cm.resources | length > 0 - -- name: Write cluster CA to file - ansible.builtin.copy: - content: "{{ ca_content }}" - dest: "{{ output_path }}" - mode: "0644" - when: ca_content is defined and ca_content != '' - -- name: Stat extracted CA output - ansible.builtin.stat: - path: "{{ output_path }}" - register: odf_ssl_ca_out_stat - -- name: Fail when CA file missing or too small after extract - ansible.builtin.fail: - msg: >- - Extraction produced no usable PEM at {{ output_path }} (cluster={{ cluster_name | default('unknown') }}). - Source: ConfigMap trusted-ca-bundle, namespace openshift-config-managed - (hub: in-cluster API; spoke: kubeconfig={{ kubeconfig_path | default('(hub)') }}). - k8s_info resource count={{ ca_cm.resources | default([]) | length }}. - when: >- - not odf_ssl_ca_out_stat.stat.exists | default(false) - or (odf_ssl_ca_out_stat.stat.size | default(0) | int) < 64 diff --git a/scripts/ansible/tasks/wait-dr-managedclusters-available.yml b/scripts/ansible/tasks/wait-dr-managedclusters-available.yml deleted file mode 100644 index f8eb946..0000000 --- a/scripts/ansible/tasks/wait-dr-managedclusters-available.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# Hub context: wait until PRIMARY_CLUSTER and SECONDARY_CLUSTER exist and report -# ManagedClusterConditionAvailable=True (same semantics as tasks/kubeconfig.yml). -# Expects: primary_cluster, secondary_cluster, cluster_readiness_max_attempts, cluster_readiness_sleep. -- name: Wait until both DR ManagedClusters are Available on the hub - kubernetes.core.k8s_info: - api_version: cluster.open-cluster-management.io/v1 - kind: ManagedCluster - register: mc_wait_ready - until: >- - ( - ( - ( - ( - mc_wait_ready.resources | default([]) - | selectattr('metadata.name', 'equalto', primary_cluster) - | list - | first - | default({}) - ).status | default({}) - ).conditions | default([]) - | selectattr('type', 'equalto', 'ManagedClusterConditionAvailable') - | map(attribute='status') - | first - | default('') - ) == 'True' - ) - and - ( - ( - ( - ( - mc_wait_ready.resources | default([]) - | selectattr('metadata.name', 'equalto', secondary_cluster) - | list - | first - | default({}) - ).status | default({}) - ).conditions | default([]) - | selectattr('type', 'equalto', 'ManagedClusterConditionAvailable') - | map(attribute='status') - | first - | default('') - ) == 'True' - ) - retries: "{{ cluster_readiness_max_attempts | int }}" - delay: "{{ cluster_readiness_sleep | int }}" - -- name: Log ManagedCluster readiness (post-wait) - ansible.builtin.debug: - msg: >- - Both ManagedClusters Available: primary={{ primary_cluster }} secondary={{ secondary_cluster }} - (ManagedClusterConditionAvailable=True on hub). diff --git a/scripts/odf-ssl-certificate-extraction.sh b/scripts/odf-ssl-certificate-extraction.sh deleted file mode 100755 index bef5262..0000000 --- a/scripts/odf-ssl-certificate-extraction.sh +++ /dev/null @@ -1,709 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "Starting ODF SSL certificate extraction and distribution..." -echo "Following Red Hat ODF Disaster Recovery certificate management guidelines" - -# Configuration for retry logic -MAX_RETRIES=5 -BASE_DELAY=30 -MAX_DELAY=300 -RETRY_COUNT=0 - -array_contains() { - local needle="$1" - shift - local item - for item in "$@"; do - [[ "$item" == "$needle" ]] && return 0 - done - return 1 -} - -# Function to implement exponential backoff -exponential_backoff() { - local delay=$((BASE_DELAY * (2 ** RETRY_COUNT))) - if [[ $delay -gt $MAX_DELAY ]]; then - delay=$MAX_DELAY - fi - echo "โณ Waiting $delay seconds before retry (attempt $((RETRY_COUNT + 1))/$MAX_RETRIES)..." - sleep $delay - ((RETRY_COUNT++)) -} - -# Function to handle errors gracefully -handle_error() { - local error_msg="$1" - echo "โŒ Error: $error_msg" - - if [[ $RETRY_COUNT -lt $MAX_RETRIES ]]; then - echo "๐Ÿ”„ Retrying in a moment..." - exponential_backoff - return 0 - else - echo "๐Ÿ’ฅ Max retries exceeded. Job will exit but ArgoCD can retry the sync." - echo " This is a temporary failure - the job will be retried on next ArgoCD sync." - exit 1 - fi -} - -# Main execution with retry logic -main_execution() { - # Create working directory - WORK_DIR="/tmp/odf-ssl-certs" - mkdir -p "$WORK_DIR" - -# Function to extract CA from cluster -extract_cluster_ca() { - cluster_name="$1" - output_file="$2" - kubeconfig="${3:-}" - - echo "Extracting CA from cluster: $cluster_name" - - if [[ -n "$kubeconfig" && -f "$kubeconfig" ]]; then - # Use provided kubeconfig - echo " Using kubeconfig: $kubeconfig" - if oc --kubeconfig="$kubeconfig" get configmap -n openshift-config-managed trusted-ca-bundle -o jsonpath="{.data['ca-bundle\.crt']}" > "$output_file" 2>/dev/null; then - if [[ -s "$output_file" ]]; then - echo " CA extracted from $cluster_name using kubeconfig" - return 0 - else - echo " CA file is empty from $cluster_name" - return 1 - fi - else - echo " Failed to get trusted-ca-bundle from $cluster_name" - return 1 - fi - else - # Use current context (hub cluster) - echo " Using current context for hub cluster" - if oc get configmap -n openshift-config-managed trusted-ca-bundle -o jsonpath="{.data['ca-bundle\.crt']}" > "$output_file" 2>/dev/null; then - if [[ -s "$output_file" ]]; then - echo " CA extracted from $cluster_name using current context" - return 0 - else - echo " CA file is empty from $cluster_name" - return 1 - fi - else - echo " Failed to get trusted-ca-bundle from $cluster_name" - return 1 - fi - fi -} - -# Function to extract ingress CA from cluster -extract_ingress_ca() { - cluster_name="$1" - output_file="$2" - kubeconfig="${3:-}" - - echo "Extracting ingress CA from cluster: $cluster_name" - - if [[ -n "$kubeconfig" && -f "$kubeconfig" ]]; then - # Use provided kubeconfig - echo " Using kubeconfig: $kubeconfig" - # Try to get ingress CA from router-ca secret - if oc --kubeconfig="$kubeconfig" get secret -n openshift-ingress-operator router-ca -o jsonpath='{.data.tls\.crt}' 2>/dev/null | base64 -d > "$output_file" 2>/dev/null; then - if [[ -s "$output_file" ]]; then - echo " Ingress CA extracted from $cluster_name using kubeconfig" - return 0 - fi - fi - # Fallback: try to get from ingress operator config - if oc --kubeconfig="$kubeconfig" get secret -n openshift-ingress-operator router-ca -o jsonpath='{.data.ca\.crt}' 2>/dev/null | base64 -d > "$output_file" 2>/dev/null; then - if [[ -s "$output_file" ]]; then - echo " Ingress CA extracted from $cluster_name using kubeconfig (fallback)" - return 0 - fi - fi - echo " Failed to get ingress CA from $cluster_name" - return 1 - else - # Use current context (hub cluster) - echo " Using current context for hub cluster" - # Try to get ingress CA from router-ca secret - if oc get secret -n openshift-ingress-operator router-ca -o jsonpath='{.data.tls\.crt}' 2>/dev/null | base64 -d > "$output_file" 2>/dev/null; then - if [[ -s "$output_file" ]]; then - echo " Ingress CA extracted from $cluster_name using current context" - return 0 - fi - fi - # Fallback: try to get from ingress operator config - if oc get secret -n openshift-ingress-operator router-ca -o jsonpath='{.data.ca\.crt}' 2>/dev/null | base64 -d > "$output_file" 2>/dev/null; then - if [[ -s "$output_file" ]]; then - echo " Ingress CA extracted from $cluster_name using current context (fallback)" - return 0 - fi - fi - echo " Failed to get ingress CA from $cluster_name" - return 1 - fi -} - -# Function to create combined CA bundle -create_combined_ca_bundle() { - output_file="$1" - shift - ca_files=("$@") - - echo "Creating combined CA bundle..." - : > "$output_file" - - file_count=0 - for ca_file in "${ca_files[@]}"; do - if [[ -f "$ca_file" && -s "$ca_file" ]]; then - echo "# CA from $(basename "$ca_file" .crt)" >> "$output_file" - - # Extract only the first few complete certificates to avoid size limits - cert_count=0 - in_cert=false - while IFS= read -r line; do - if [[ "$line" == "-----BEGIN CERTIFICATE-----" ]]; then - in_cert=true - cert_count=$((cert_count + 1)) - if [[ $cert_count -gt 5 ]]; then - break - fi - fi - if [[ $in_cert == true ]]; then - echo "$line" >> "$output_file" - fi - if [[ "$line" == "-----END CERTIFICATE-----" ]]; then - in_cert=false - echo "" >> "$output_file" - fi - done < "$ca_file" - - file_count=$((file_count + 1)) - fi - done - - if [[ $file_count -gt 0 ]]; then - echo "Combined CA bundle created with $file_count CA sources (first 5 certs each)" - return 0 - else - echo "No valid CA files found to combine" - return 1 - fi -} - -# Extract hub cluster CA -echo "1. Extracting hub cluster CA..." -if extract_cluster_ca "hub" "$WORK_DIR/hub-ca.crt"; then - echo " Hub cluster CA extracted successfully" - echo " Certificate size: $(wc -c < "$WORK_DIR/hub-ca.crt") bytes" - echo " First few lines:" - head -n 5 "$WORK_DIR/hub-ca.crt" -else - echo " Failed to extract hub cluster CA" - echo " Job will continue with managed cluster certificates only" -fi - -# Extract hub cluster ingress CA -echo "1b. Extracting hub cluster ingress CA..." -if extract_ingress_ca "hub" "$WORK_DIR/hub-ingress-ca.crt"; then - echo " Hub cluster ingress CA extracted successfully" - echo " Certificate size: $(wc -c < "$WORK_DIR/hub-ingress-ca.crt") bytes" -else - echo " Failed to extract hub cluster ingress CA" - echo " Job will continue without hub ingress CA" -fi - -# Get managed clusters -echo "2. Discovering managed clusters..." -MANAGED_CLUSTERS=$(oc get managedclusters -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - -if [[ -z "$MANAGED_CLUSTERS" ]]; then - echo " No managed clusters found" -else - echo " Found managed clusters: $MANAGED_CLUSTERS" -fi - -# Primary and secondary managed cluster names (from values.yaml via env) -PRIMARY_CLUSTER="${PRIMARY_CLUSTER:-ocp-primary}" -SECONDARY_CLUSTER="${SECONDARY_CLUSTER:-ocp-secondary}" - -# Extract CA from each managed cluster -CA_FILES=() -REQUIRED_CLUSTERS=("hub" "$PRIMARY_CLUSTER" "$SECONDARY_CLUSTER") -EXTRACTED_CLUSTERS=() - -# Track hub cluster CA extraction -if [[ -f "$WORK_DIR/hub-ca.crt" && -s "$WORK_DIR/hub-ca.crt" ]]; then - CA_FILES+=("$WORK_DIR/hub-ca.crt") - EXTRACTED_CLUSTERS+=("hub") - echo " Added hub CA to bundle" -else - echo " โŒ Hub CA not available - REQUIRED for DR setup" -fi - -if [[ -f "$WORK_DIR/hub-ingress-ca.crt" && -s "$WORK_DIR/hub-ingress-ca.crt" ]]; then - CA_FILES+=("$WORK_DIR/hub-ingress-ca.crt") - echo " Added hub ingress CA to bundle" -else - echo " Hub ingress CA not available, continuing without it" -fi - -index=1 - -for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo "3.$index Extracting CA from $cluster..." - - # Try to get kubeconfig for the cluster - KUBECONFIG_FILE="" - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "$WORK_DIR/${cluster}-kubeconfig.yaml" 2>/dev/null; then - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - fi - - if extract_cluster_ca "$cluster" "$WORK_DIR/${cluster}-ca.crt" "$KUBECONFIG_FILE"; then - CA_FILES+=("$WORK_DIR/${cluster}-ca.crt") - EXTRACTED_CLUSTERS+=("$cluster") - echo " Certificate size: $(wc -c < "$WORK_DIR/${cluster}-ca.crt") bytes" - else - echo " โŒ Could not extract CA from $cluster - REQUIRED for DR setup" - fi - - # Extract ingress CA from managed cluster - echo "3b.$index Extracting ingress CA from $cluster..." - if extract_ingress_ca "$cluster" "$WORK_DIR/${cluster}-ingress-ca.crt" "$KUBECONFIG_FILE"; then - CA_FILES+=("$WORK_DIR/${cluster}-ingress-ca.crt") - echo " Ingress CA certificate size: $(wc -c < "$WORK_DIR/${cluster}-ingress-ca.crt") bytes" - else - echo " Warning: Could not extract ingress CA from $cluster, continuing without it" - fi - - ((index++)) -done - -# Validate that we have CA material from all required clusters -echo "4. Validating CA extraction from required clusters..." -MISSING_CLUSTERS=() -for required_cluster in "${REQUIRED_CLUSTERS[@]}"; do - if array_contains "$required_cluster" "${EXTRACTED_CLUSTERS[@]}"; then - echo " โœ… CA extracted from $required_cluster" - else - echo " โŒ CA NOT extracted from $required_cluster" - MISSING_CLUSTERS+=("$required_cluster") - fi -done - -if [[ ${#MISSING_CLUSTERS[@]} -gt 0 ]]; then - echo "" - echo "โŒ CRITICAL ERROR: CA material missing from required clusters:" - for missing in "${MISSING_CLUSTERS[@]}"; do - echo " - $missing" - done - echo "" - echo "The ODF SSL certificate extractor job requires CA material from ALL three clusters:" - echo " - hub (hub cluster)" - echo " - $PRIMARY_CLUSTER (primary managed cluster)" - echo " - $SECONDARY_CLUSTER (secondary managed cluster)" - echo "" - echo "Without CA material from all clusters, the DR setup will fail." - echo "Please ensure all clusters are accessible and have proper kubeconfigs." - echo "" - echo "Job will exit with error code 1." - exit 1 -fi - -# Create combined CA bundle -echo "5. Creating combined CA bundle..." -echo " CA files to combine: ${#CA_FILES[@]} files" -for ca_file in "${CA_FILES[@]}"; do - echo " - $(basename "$ca_file") ($(wc -c < "$ca_file") bytes)" -done - -if create_combined_ca_bundle "$WORK_DIR/combined-ca-bundle.crt" "${CA_FILES[@]}"; then - echo " Combined CA bundle created successfully" - echo " Bundle size: $(wc -c < "$WORK_DIR/combined-ca-bundle.crt") bytes" - echo " First few lines of bundle:" - head -n 10 "$WORK_DIR/combined-ca-bundle.crt" -else - echo " Failed to create combined CA bundle - no certificates extracted" - echo " Job will exit as no certificate data is available" - exit 1 -fi - -# Create or update ConfigMap on hub cluster -echo "6. Creating/updating cluster-proxy-ca-bundle ConfigMap on hub cluster..." - -# Check if ConfigMap exists -if oc get configmap cluster-proxy-ca-bundle -n openshift-config >/dev/null 2>&1; then - echo " ConfigMap exists, patching with certificate data..." - # Create a temporary patch file to avoid JSON escaping issues - echo "data:" > "$WORK_DIR/patch.yaml" - echo " ca-bundle.crt: |" >> "$WORK_DIR/patch.yaml" - cat "$WORK_DIR/combined-ca-bundle.crt" | sed 's/^/ /' >> "$WORK_DIR/patch.yaml" - oc patch configmap cluster-proxy-ca-bundle -n openshift-config \ - --type=merge \ - --patch-file="$WORK_DIR/patch.yaml" - rm -f "$WORK_DIR/patch.yaml" -else - echo " ConfigMap does not exist, creating with certificate data..." - oc create configmap cluster-proxy-ca-bundle \ - --from-file=ca-bundle.crt="$WORK_DIR/combined-ca-bundle.crt" \ - -n openshift-config -fi - -echo " ConfigMap created/updated successfully with certificate data" -echo " Certificate bundle contains CA certificates from hub and managed clusters" - -# Update hub cluster proxy -echo "7. Updating hub cluster proxy configuration..." -oc patch proxy/cluster --type=merge --patch='{"spec":{"trustedCA":{"name":"cluster-proxy-ca-bundle"}}}' || { - echo " Warning: Could not update hub cluster proxy" -} - -# Restart ramenddr-cluster-operator pods on managed clusters -echo "7a. Restarting ramenddr-cluster-operator pods on managed clusters..." - -for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Processing cluster: $cluster" - - # Get kubeconfig for the cluster - KUBECONFIG_FILE="" - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "$WORK_DIR/${cluster}-kubeconfig.yaml" 2>/dev/null; then - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - fi - - if [[ -n "$KUBECONFIG_FILE" && -f "$KUBECONFIG_FILE" ]]; then - # Find ramenddr-cluster-operator pods - RAMEN_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-dr-system -l app=ramenddr-cluster-operator -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - if [[ -n "$RAMEN_PODS" ]]; then - echo " Found ramenddr-cluster-operator pods: $RAMEN_PODS" - - for pod in $RAMEN_PODS; do - echo " Deleting pod $pod to trigger restart..." - oc --kubeconfig="$KUBECONFIG_FILE" delete pod "$pod" -n openshift-dr-system --ignore-not-found=true || { - echo " Warning: Could not delete pod $pod" - } - done - - # Wait for pods to be deleted - echo " Waiting for pods to be terminated..." - for pod in $RAMEN_PODS; do - oc --kubeconfig="$KUBECONFIG_FILE" wait --for=delete pod/"$pod" -n openshift-dr-system --timeout=60s 2>/dev/null || true - done - - # Wait for new pods to be running - echo " Waiting for new ramenddr-cluster-operator pods to be running..." - MAX_WAIT_ATTEMPTS=30 - WAIT_INTERVAL=10 - attempt=0 - - while [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; do - attempt=$((attempt + 1)) - - NEW_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-dr-system -l app=ramenddr-cluster-operator -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - ALL_RUNNING=true - - if [[ -n "$NEW_PODS" ]]; then - for pod in $NEW_PODS; do - POD_STATUS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pod "$pod" -n openshift-dr-system -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - - if [[ "$POD_STATUS" != "Running" ]]; then - ALL_RUNNING=false - break - fi - done - - if [[ "$ALL_RUNNING" == "true" ]]; then - echo " โœ… All ramenddr-cluster-operator pods are running on $cluster: $NEW_PODS" - break - else - echo " โณ Waiting for pods to be running (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - else - echo " โณ Waiting for pods to appear (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - - if [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; then - sleep $WAIT_INTERVAL - fi - done - - if [[ $attempt -ge $MAX_WAIT_ATTEMPTS ]]; then - echo " โš ๏ธ Warning: ramenddr-cluster-operator pods did not become ready within expected time on $cluster" - echo " The pods may still be starting - configuration changes will be applied when ready" - fi - else - echo " โš ๏ธ Warning: ramenddr-cluster-operator pods not found on $cluster - they may not be deployed yet" - echo " Configuration changes will be applied when the pods start" - fi - else - echo " โŒ Could not get kubeconfig for $cluster - skipping pod restart" - fi -done - -echo " โœ… Completed ramenddr-cluster-operator pod restarts on managed clusters" - -# ramen-hub-operator-config caCertificates (s3StoreProfiles) are owned by regional DR / Ramen charts โ€” not this extractor. - -# Restart Velero pods on managed clusters to pick up new CA certificates -echo "7c. Restarting Velero pods on managed clusters..." - -for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Processing cluster: $cluster" - - # Get kubeconfig for the cluster - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - if [[ ! -f "$KUBECONFIG_FILE" ]]; then - # Fetch kubeconfig if not already available - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "$KUBECONFIG_FILE" 2>/dev/null; then - echo " Fetched kubeconfig for $cluster" - else - echo " โŒ Could not get kubeconfig for $cluster - skipping Velero pod restart" - continue - fi - fi - - # Find Velero pods in openshift-adp namespace - VELERO_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-adp -l component=velero -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - if [[ -n "$VELERO_PODS" ]]; then - echo " Found Velero pods: $VELERO_PODS" - - for pod in $VELERO_PODS; do - echo " Deleting pod $pod to trigger restart..." - oc --kubeconfig="$KUBECONFIG_FILE" delete pod "$pod" -n openshift-adp --ignore-not-found=true || { - echo " Warning: Could not delete pod $pod" - } - done - - # Wait for pods to be deleted - echo " Waiting for pods to be terminated..." - for pod in $VELERO_PODS; do - oc --kubeconfig="$KUBECONFIG_FILE" wait --for=delete pod/"$pod" -n openshift-adp --timeout=60s 2>/dev/null || true - done - - # Wait for new pods to be running - echo " Waiting for new Velero pods to be running..." - MAX_WAIT_ATTEMPTS=30 - WAIT_INTERVAL=10 - attempt=0 - - while [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; do - attempt=$((attempt + 1)) - - NEW_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-adp -l component=velero -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - ALL_RUNNING=true - - if [[ -n "$NEW_PODS" ]]; then - for pod in $NEW_PODS; do - POD_STATUS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pod "$pod" -n openshift-adp -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - - if [[ "$POD_STATUS" != "Running" ]]; then - ALL_RUNNING=false - break - fi - done - - if [[ "$ALL_RUNNING" == "true" ]]; then - echo " โœ… All Velero pods are running on $cluster: $NEW_PODS" - break - else - echo " โณ Waiting for pods to be running (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - else - echo " โณ Waiting for pods to appear (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - - if [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; then - sleep $WAIT_INTERVAL - fi - done - - if [[ $attempt -ge $MAX_WAIT_ATTEMPTS ]]; then - echo " โš ๏ธ Warning: Velero pods did not become ready within expected time on $cluster" - echo " The pods may still be starting - new CA certificates will be applied when ready" - fi - else - echo " โš ๏ธ Warning: Velero pods not found on $cluster - they may not be deployed yet" - echo " New CA certificates will be applied when the pods start" - fi -done - -echo " โœ… Completed Velero pod restarts on managed clusters" - -# Distribute certificate data to managed clusters with retry logic -echo "8. Distributing certificate data to managed clusters..." -DISTRIBUTION_ATTEMPTS=3 -DISTRIBUTION_SLEEP=10 - -for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Distributing to $cluster..." - - # Get kubeconfig for the cluster - KUBECONFIG_FILE="" - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "$WORK_DIR/${cluster}-kubeconfig.yaml" 2>/dev/null; then - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - fi - - if [[ -n "$KUBECONFIG_FILE" && -f "$KUBECONFIG_FILE" ]]; then - # Retry distribution to managed cluster - distribution_success=false - for dist_attempt in $(seq 1 $DISTRIBUTION_ATTEMPTS); do - echo " Distribution attempt $dist_attempt/$DISTRIBUTION_ATTEMPTS for $cluster..." - - # Create ConfigMap on managed cluster - if oc --kubeconfig="$KUBECONFIG_FILE" create configmap cluster-proxy-ca-bundle \ - --from-file=ca-bundle.crt="$WORK_DIR/combined-ca-bundle.crt" \ - -n openshift-config \ - --dry-run=client -o yaml | oc --kubeconfig="$KUBECONFIG_FILE" apply -f -; then - - # Update managed cluster proxy - if oc --kubeconfig="$KUBECONFIG_FILE" patch proxy/cluster --type=merge --patch='{"spec":{"trustedCA":{"name":"cluster-proxy-ca-bundle"}}}'; then - echo " โœ… Certificate data distributed to $cluster (attempt $dist_attempt)" - distribution_success=true - break - else - echo " โš ๏ธ ConfigMap created but proxy update failed for $cluster (attempt $dist_attempt)" - fi - else - echo " โš ๏ธ ConfigMap creation failed for $cluster (attempt $dist_attempt)" - fi - - if [[ $dist_attempt -lt $DISTRIBUTION_ATTEMPTS ]]; then - echo " โณ Waiting $DISTRIBUTION_SLEEP seconds before retry..." - sleep $DISTRIBUTION_SLEEP - fi - done - - if [[ "$distribution_success" != "true" ]]; then - echo " โŒ Failed to distribute certificate data to $cluster after $DISTRIBUTION_ATTEMPTS attempts" - echo " This may cause DR prerequisites check to fail" - fi - else - echo " โŒ Could not get kubeconfig for $cluster - skipping distribution" - fi -done - -# Verify distribution to managed clusters -echo "9. Verifying certificate distribution to managed clusters..." -verification_failed=false -REQUIRED_VERIFICATION_CLUSTERS=("$PRIMARY_CLUSTER" "$SECONDARY_CLUSTER") -VERIFIED_CLUSTERS=() - -for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Verifying distribution to $cluster..." - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - - if [[ -f "$KUBECONFIG_FILE" ]]; then - # Check if ConfigMap exists and has content - configmap_exists=$(oc --kubeconfig="$KUBECONFIG_FILE" get configmap cluster-proxy-ca-bundle -n openshift-config &>/dev/null && echo "true" || echo "false") - configmap_size=$(oc --kubeconfig="$KUBECONFIG_FILE" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c || echo "0") - proxy_configured=$(oc --kubeconfig="$KUBECONFIG_FILE" get proxy cluster -o jsonpath='{.spec.trustedCA.name}' 2>/dev/null || echo "") - - if [[ "$configmap_exists" == "true" && $configmap_size -gt 100 && "$proxy_configured" == "cluster-proxy-ca-bundle" ]]; then - echo " โœ… $cluster: ConfigMap exists (${configmap_size} bytes), proxy configured" - VERIFIED_CLUSTERS+=("$cluster") - else - echo " โŒ $cluster: ConfigMap verification failed" - echo " ConfigMap exists: $configmap_exists" - echo " ConfigMap size: $configmap_size bytes" - echo " Proxy configured: $proxy_configured" - verification_failed=true - fi - else - echo " โŒ $cluster: No kubeconfig available for verification" - verification_failed=true - fi -done - -# Check if all required clusters are verified -echo "10. Validating verification results..." -MISSING_VERIFICATION_CLUSTERS=() -for required_cluster in "${REQUIRED_VERIFICATION_CLUSTERS[@]}"; do - if array_contains "$required_cluster" "${VERIFIED_CLUSTERS[@]}"; then - echo " โœ… $required_cluster: Certificate distribution verified" - else - echo " โŒ $required_cluster: Certificate distribution NOT verified" - MISSING_VERIFICATION_CLUSTERS+=("$required_cluster") - fi -done - -if [[ ${#MISSING_VERIFICATION_CLUSTERS[@]} -gt 0 ]]; then - echo "" - echo "โŒ CRITICAL ERROR: Certificate distribution verification failed for required clusters:" - for missing in "${MISSING_VERIFICATION_CLUSTERS[@]}"; do - echo " - $missing" - done - echo "" - echo "The ODF SSL certificate extractor job requires successful certificate distribution" - echo "to ALL managed clusters ($PRIMARY_CLUSTER and $SECONDARY_CLUSTER)." - echo "" - echo "Without proper certificate distribution, the DR setup will fail." - echo "Please check cluster connectivity and kubeconfig availability." - echo "" - echo "Job will exit with error code 1." - exit 1 -fi - -if [[ "$verification_failed" == "true" ]]; then - echo "" - echo "โš ๏ธ Certificate distribution verification failed for some clusters" - echo " This may cause DR prerequisites check to fail" - echo " Manual intervention may be required" - echo "" - echo "Job will exit with error code 1." - exit 1 -else - echo "" - echo "โœ… All managed clusters verified successfully" -fi - - -echo "" -echo "โœ… ODF SSL certificate management completed successfully!" -echo " - Hub cluster CA bundle: Updated (includes trusted CA + ingress CA)" -echo " - Hub cluster proxy: Configured" -echo " - Managed clusters: ramenddr-cluster-operator pods restarted" -echo " - Managed clusters: Velero pods restarted (openshift-adp namespace)" -echo " - Managed clusters: Certificate data distributed (includes ingress CAs)" -echo "" -echo "This follows Red Hat ODF Disaster Recovery certificate management guidelines" -echo "for secure SSL access across clusters in the regional DR setup." -echo "Ramen hub s3StoreProfiles / caCertificates are configured by regional DR or Ramen charts, not this job." -} - -# Execute main function with retry logic -while true; do - if main_execution; then - echo "๐ŸŽ‰ Certificate extraction completed successfully!" - exit 0 - else - if [[ $RETRY_COUNT -lt $MAX_RETRIES ]]; then - echo "๐Ÿ”„ Main execution failed, retrying..." - exponential_backoff - continue - else - echo "๐Ÿ’ฅ Max retries exceeded. Job will exit but ArgoCD can retry the sync." - echo " This is a temporary failure - the job will be retried on next ArgoCD sync." - exit 1 - fi - fi -done diff --git a/scripts/odf-ssl-precheck.sh b/scripts/odf-ssl-precheck.sh deleted file mode 100755 index 473309c..0000000 --- a/scripts/odf-ssl-precheck.sh +++ /dev/null @@ -1,908 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "Starting ODF SSL certificate precheck and distribution..." -echo "This job ensures certificates are properly distributed before DR policies are applied" - -# Configuration -CLUSTER_CA_MGT_NAMESPACE="${CLUSTER_CA_MGT_NAMESPACE:-cluster-ca-mgt}" -MIN_CERTIFICATES=15 -MIN_BUNDLE_SIZE=20000 -MAX_ATTEMPTS=120 -SLEEP_INTERVAL=30 -CLUSTER_READINESS_MAX_ATTEMPTS=150 # Wait up to 75 minutes for clusters to be ready (150 * 30s) -CLUSTER_READINESS_SLEEP=30 - -# Function to clean up placeholder ConfigMaps -cleanup_placeholder_configmaps() { - echo "๐Ÿงน Cleaning up placeholder ConfigMaps from managed clusters..." - - MANAGED_CLUSTERS=$(oc get managedclusters -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - if [[ -z "$MANAGED_CLUSTERS" ]]; then - echo "No managed clusters found" - return 1 - fi - - for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo "Checking $cluster for placeholder ConfigMaps..." - - KUBECONFIG_FILE="" - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "/tmp/${cluster}-kubeconfig.yaml" 2>/dev/null; then - KUBECONFIG_FILE="/tmp/${cluster}-kubeconfig.yaml" - fi - - if [[ -n "$KUBECONFIG_FILE" && -f "$KUBECONFIG_FILE" ]]; then - configmap_content=$(oc --kubeconfig="$KUBECONFIG_FILE" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - - if [[ "$configmap_content" == *"Placeholder for ODF SSL certificate bundle"* ]] || [[ "$configmap_content" == *"This will be populated by the certificate extraction job"* ]]; then - echo " ๐Ÿ—‘๏ธ Deleting placeholder ConfigMap from $cluster..." - oc --kubeconfig="$KUBECONFIG_FILE" delete configmap cluster-proxy-ca-bundle -n openshift-config --ignore-not-found=true - echo " โœ… Placeholder ConfigMap removed from $cluster" - else - echo " โœ… $cluster: No placeholder ConfigMap found" - fi - else - echo " โŒ $cluster: Could not get kubeconfig for cleanup" - fi - done - - echo "โœ… Placeholder ConfigMap cleanup completed" - return 0 -} - -# Primary and secondary managed cluster names (from values.yaml via env) -PRIMARY_CLUSTER="${PRIMARY_CLUSTER:-ocp-primary}" -SECONDARY_CLUSTER="${SECONDARY_CLUSTER:-ocp-secondary}" - -# Function to wait for required clusters to be available and joined -wait_for_cluster_readiness() { - echo "๐Ÿ” Waiting for required clusters ($PRIMARY_CLUSTER and $SECONDARY_CLUSTER) to be available and joined..." - echo " This may take several minutes during initial cluster deployment" - - REQUIRED_CLUSTERS=("$PRIMARY_CLUSTER" "$SECONDARY_CLUSTER") - attempt=1 - - while [[ $attempt -le $CLUSTER_READINESS_MAX_ATTEMPTS ]]; do - echo "=== Cluster Readiness Check Attempt $attempt/$CLUSTER_READINESS_MAX_ATTEMPTS ===" - - all_ready=true - unready_clusters=() - - for cluster in "${REQUIRED_CLUSTERS[@]}"; do - # Check if cluster exists - if ! oc get managedcluster "$cluster" &>/dev/null; then - echo " โณ Cluster $cluster does not exist yet..." - all_ready=false - unready_clusters+=("$cluster") - continue - fi - - # Check if cluster is available - cluster_status=$(oc get managedcluster "$cluster" -o jsonpath='{.status.conditions[?(@.type=="ManagedClusterConditionAvailable")].status}' 2>/dev/null || echo "Unknown") - if [[ "$cluster_status" != "True" ]]; then - echo " โณ Cluster $cluster is not available yet (status: $cluster_status)" - all_ready=false - unready_clusters+=("$cluster") - continue - fi - - # Check if cluster is joined - joined_status=$(oc get managedcluster "$cluster" -o jsonpath='{.status.conditions[?(@.type=="ManagedClusterJoined")].status}' 2>/dev/null || echo "Unknown") - if [[ "$joined_status" != "True" ]]; then - echo " โณ Cluster $cluster is not joined yet (status: $joined_status)" - all_ready=false - unready_clusters+=("$cluster") - continue - fi - - echo " โœ… Cluster $cluster is available and joined" - done - - if [[ "$all_ready" == "true" ]]; then - echo "โœ… All required clusters are available and joined!" - return 0 - else - echo "โณ Waiting for clusters to be ready: ${unready_clusters[*]}" - echo " This is normal during initial cluster deployment - clusters may take 10-30 minutes to become ready" - - if [[ $attempt -ge $CLUSTER_READINESS_MAX_ATTEMPTS ]]; then - echo "โŒ TIMEOUT: Clusters are still not ready after $CLUSTER_READINESS_MAX_ATTEMPTS attempts ($((CLUSTER_READINESS_MAX_ATTEMPTS * CLUSTER_READINESS_SLEEP / 60)) minutes)" - echo " Unready clusters: ${unready_clusters[*]}" - echo " This may indicate a problem with cluster deployment" - echo " The precheck will continue but certificate extraction may fail" - return 1 - else - sleep $CLUSTER_READINESS_SLEEP - ((attempt++)) - fi - fi - done - - return 1 -} - -# Function to check certificate distribution -check_certificate_distribution() { - echo "Checking certificate distribution status..." - - if ! oc get configmap cluster-proxy-ca-bundle -n openshift-config >/dev/null 2>&1; then - echo "โŒ CA bundle ConfigMap not found on hub cluster" - return 1 - fi - - bundle_content=$(oc get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath="{.data['ca-bundle\.crt']}" 2>/dev/null || echo "") - - if [[ -z "$bundle_content" ]]; then - echo "โŒ CA bundle is empty" - return 1 - fi - - bundle_size=${#bundle_content} - echo " Bundle size: $bundle_size bytes" - - if [[ $bundle_size -lt $MIN_BUNDLE_SIZE ]]; then - echo "โŒ CA bundle too small ($bundle_size < $MIN_BUNDLE_SIZE bytes)" - return 1 - fi - - cert_count=$(echo "$bundle_content" | grep -c "BEGIN CERTIFICATE" || echo "0") - echo " Certificate count: $cert_count" - - if [[ $cert_count -lt $MIN_CERTIFICATES ]]; then - echo "โŒ Too few certificates ($cert_count < $MIN_CERTIFICATES)" - return 1 - fi - - hub_certs=$(echo "$bundle_content" | grep -c "hub" || echo "0") - ocp_primary_certs=$(echo "$bundle_content" | grep -c "$PRIMARY_CLUSTER" || echo "0") - ocp_secondary_certs=$(echo "$bundle_content" | grep -c "$SECONDARY_CLUSTER" || echo "0") - - echo " Hub cluster certificates: $hub_certs" - echo " $PRIMARY_CLUSTER certificates: $ocp_primary_certs" - echo " $SECONDARY_CLUSTER certificates: $ocp_secondary_certs" - - if [[ $hub_certs -lt 2 || $ocp_primary_certs -lt 2 || $ocp_secondary_certs -lt 2 ]]; then - echo "โŒ Missing certificates from one or more clusters" - return 1 - fi - - echo "โœ… CA bundle is complete and properly distributed" - return 0 -} - -# Function to trigger certificate extraction -# shellcheck disable=SC2120 -trigger_certificate_extraction() { - echo "Triggering certificate extraction..." - - oc delete job odf-ssl-certificate-extractor -n "$CLUSTER_CA_MGT_NAMESPACE" --ignore-not-found=true - sleep 5 - - echo "Creating certificate extraction job..." - # shellcheck disable=SC2154,SC2076,SC2000,SC2012,SC2035,SC2086 - oc apply -f - < "$output_file" - echo " CA extracted from $cluster_name using kubeconfig" - else - oc get configmap -n openshift-config-managed trusted-ca-bundle -o jsonpath="{.data['ca-bundle\.crt']}" > "$output_file" - echo " CA extracted from $cluster_name using current context" - fi - - cert_size=$(wc -c < "$output_file" 2>/dev/null || echo "0") - echo " Certificate size: $cert_size bytes" - - if [[ $cert_size -lt 1000 ]]; then - echo " Warning: Certificate size seems too small" - return 1 - fi - - return 0 - } - - extract_ingress_ca() { - cluster_name="$1" - output_file="$2" - kubeconfig="${3:-}" - - echo "Extracting ingress CA from cluster: $cluster_name" - - if [[ -n "$kubeconfig" && -f "$kubeconfig" ]]; then - KUBECONFIG="$kubeconfig" oc get configmap -n openshift-config-managed router-ca -o jsonpath="{.data['ca-bundle\.crt']}" > "$output_file" 2>/dev/null || echo "" > "$output_file" - echo " Ingress CA extracted from $cluster_name using kubeconfig" - else - oc get configmap -n openshift-config-managed router-ca -o jsonpath="{.data['ca-bundle\.crt']}" > "$output_file" 2>/dev/null || echo "" > "$output_file" - echo " Ingress CA extracted from $cluster_name using current context" - fi - - cert_size=$(wc -c < "$output_file" 2>/dev/null || echo "0") - echo " Ingress CA certificate size: $cert_size bytes" - - return 0 - } - - create_combined_ca_bundle() { - output_file="$1" - shift - ca_files=("$@") - - echo "Creating combined CA bundle..." - > "$output_file" - - file_count=0 - for ca_file in "${ca_files[@]}"; do - if [[ -f "$ca_file" && -s "$ca_file" ]]; then - echo "# CA from $(basename "$ca_file" .crt)" >> "$output_file" - - cert_count=0 - in_cert=false - while IFS= read -r line; do - if [[ "$line" == "-----BEGIN CERTIFICATE-----" ]]; then - in_cert=true - cert_count=$((cert_count + 1)) - if [[ $cert_count -gt 5 ]]; then - break - fi - fi - if [[ $in_cert == true ]]; then - echo "$line" >> "$output_file" - fi - if [[ "$line" == "-----END CERTIFICATE-----" ]]; then - in_cert=false - echo "" >> "$output_file" - fi - done < "$ca_file" - - file_count=$((file_count + 1)) - fi - done - - if [[ $file_count -gt 0 ]]; then - echo "Combined CA bundle created with $file_count CA sources (first 5 certs each)" - return 0 - else - echo "No valid CA files found to combine" - return 1 - fi - } - - echo "1. Extracting hub cluster CA..." - hub_ca_extracted=false - if extract_cluster_ca "hub" ""; then - hub_ca_extracted=true - echo " โœ… Hub CA extracted successfully" - else - echo " โŒ Hub CA extraction failed - REQUIRED for DR setup" - fi - - extract_ingress_ca "hub" "" - - echo "2. Discovering managed clusters..." - managed_clusters=$(oc get managedclusters -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -v local-cluster || echo "") - echo " Found managed clusters: $managed_clusters" - - echo " Added hub CA to bundle" - echo " Added hub ingress CA to bundle" - - # Track required clusters - REQUIRED_CLUSTERS=("hub" "$PRIMARY_CLUSTER" "$SECONDARY_CLUSTER") - EXTRACTED_CLUSTERS=() - if [[ "$hub_ca_extracted" == "true" ]]; then - EXTRACTED_CLUSTERS+=("hub") - fi - - cluster_count=0 - for cluster in $managed_clusters; do - if [[ "$cluster" == "$PRIMARY_CLUSTER" || "$cluster" == "$SECONDARY_CLUSTER" ]]; then - cluster_count=$((cluster_count + 1)) - echo "3.$cluster_count Extracting CA from $cluster..." - - kubeconfig_file="/tmp/odf-ssl-certs/${cluster}-kubeconfig.yaml" - oc get secret "${cluster}-import" -n "${cluster}" -o jsonpath="{.data.kubeconfig}" | base64 -d > "$kubeconfig_file" 2>/dev/null || { - echo " โŒ Could not get kubeconfig for $cluster - REQUIRED for DR setup" - continue - } - - if extract_cluster_ca "$cluster" "$kubeconfig_file"; then - EXTRACTED_CLUSTERS+=("$cluster") - echo " โœ… CA extracted from $cluster" - else - echo " โŒ CA extraction failed from $cluster - REQUIRED for DR setup" - fi - - extract_ingress_ca "$cluster" "$kubeconfig_file" - fi - done - - # Validate that we have CA material from all required clusters - echo "4. Validating CA extraction from required clusters..." - MISSING_CLUSTERS=() - for required_cluster in "${REQUIRED_CLUSTERS[@]}"; do - if [[ " ${EXTRACTED_CLUSTERS[@]} " =~ " ${required_cluster} " ]]; then - echo " โœ… CA extracted from $required_cluster" - else - echo " โŒ CA NOT extracted from $required_cluster" - MISSING_CLUSTERS+=("$required_cluster") - fi - done - - if [[ ${#MISSING_CLUSTERS[@]} -gt 0 ]]; then - echo "" - echo "โŒ CRITICAL ERROR: CA material missing from required clusters:" - for missing in "${MISSING_CLUSTERS[@]}"; do - echo " - $missing" - done - echo "" - echo "The ODF SSL certificate extractor job requires CA material from ALL three clusters:" - echo " - hub (hub cluster)" - echo " - $PRIMARY_CLUSTER (primary managed cluster)" - echo " - $SECONDARY_CLUSTER (secondary managed cluster)" - echo "" - echo "Without CA material from all clusters, the DR setup will fail." - echo "Please ensure all clusters are accessible and have proper kubeconfigs." - echo "" - echo "Job will exit with error code 1." - exit 1 - fi - - echo "5. Creating combined CA bundle..." - ca_files=$(ls -1 *.crt 2>/dev/null | wc -l) - echo " CA files to combine: $ca_files files" - - for file in *.crt; do - if [[ -f "$file" ]]; then - file_size=$(wc -c < "$file" 2>/dev/null || echo "0") - echo " - $file ($file_size bytes)" - fi - done - - create_combined_ca_bundle "combined-ca-bundle.crt" *.crt - - bundle_size=$(wc -c < combined-ca-bundle.crt) - cert_count=$(grep -c "BEGIN CERTIFICATE" combined-ca-bundle.crt || echo "0") - - echo "Combined CA bundle created with $ca_files CA sources (first 5 certs each)" - echo " Combined CA bundle created successfully" - echo " Bundle size: $bundle_size bytes" - echo " Certificate count: $cert_count" - - if [[ $bundle_size -lt 20000 ]]; then - echo "โŒ Combined CA bundle too small ($bundle_size < 20000 bytes)" - exit 1 - fi - - if [[ $cert_count -lt 15 ]]; then - echo "โŒ Too few certificates in combined CA bundle ($cert_count < 15)" - exit 1 - fi - - echo "6. Updating hub cluster ConfigMap..." - oc create configmap cluster-proxy-ca-bundle \ - --from-file=ca-bundle.crt=combined-ca-bundle.crt \ - -n openshift-config \ - --dry-run=client -o yaml | oc apply -f - - - echo " Hub cluster ConfigMap updated" - - echo "7. Updating hub cluster proxy configuration..." - oc patch proxy/cluster --type=merge --patch='{"spec":{"trustedCA":{"name":"cluster-proxy-ca-bundle"}}}' || { - echo " Warning: Could not update hub cluster proxy" - } - - # Restart ramenddr-cluster-operator pods on managed clusters - echo "7a. Restarting ramenddr-cluster-operator pods on managed clusters..." - - MANAGED_CLUSTERS=$(oc get managedclusters -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Processing cluster: $cluster" - - # Get kubeconfig for the cluster - KUBECONFIG_FILE="" - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "/tmp/${cluster}-kubeconfig.yaml" 2>/dev/null; then - KUBECONFIG_FILE="/tmp/${cluster}-kubeconfig.yaml" - fi - - if [[ -n "$KUBECONFIG_FILE" && -f "$KUBECONFIG_FILE" ]]; then - # Find ramenddr-cluster-operator pods - RAMEN_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-dr-system -l app=ramenddr-cluster-operator -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - if [[ -n "$RAMEN_PODS" ]]; then - echo " Found ramenddr-cluster-operator pods: $RAMEN_PODS" - - for pod in $RAMEN_PODS; do - echo " Deleting pod $pod to trigger restart..." - oc --kubeconfig="$KUBECONFIG_FILE" delete pod "$pod" -n openshift-dr-system --ignore-not-found=true || { - echo " Warning: Could not delete pod $pod" - } - done - - # Wait for pods to be deleted - echo " Waiting for pods to be terminated..." - for pod in $RAMEN_PODS; do - oc --kubeconfig="$KUBECONFIG_FILE" wait --for=delete pod/"$pod" -n openshift-dr-system --timeout=60s 2>/dev/null || true - done - - # Wait for new pods to be running - echo " Waiting for new ramenddr-cluster-operator pods to be running..." - MAX_WAIT_ATTEMPTS=30 - WAIT_INTERVAL=10 - attempt=0 - - while [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; do - attempt=$((attempt + 1)) - - NEW_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-dr-system -l app=ramenddr-cluster-operator -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - ALL_RUNNING=true - - if [[ -n "$NEW_PODS" ]]; then - for pod in $NEW_PODS; do - POD_STATUS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pod "$pod" -n openshift-dr-system -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - - if [[ "$POD_STATUS" != "Running" ]]; then - ALL_RUNNING=false - break - fi - done - - if [[ "$ALL_RUNNING" == "true" ]]; then - echo " โœ… All ramenddr-cluster-operator pods are running on $cluster: $NEW_PODS" - break - else - echo " โณ Waiting for pods to be running (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - else - echo " โณ Waiting for pods to appear (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - - if [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; then - sleep $WAIT_INTERVAL - fi - done - - if [[ $attempt -ge $MAX_WAIT_ATTEMPTS ]]; then - echo " โš ๏ธ Warning: ramenddr-cluster-operator pods did not become ready within expected time on $cluster" - echo " The pods may still be starting - configuration changes will be applied when ready" - fi - else - echo " โš ๏ธ Warning: ramenddr-cluster-operator pods not found on $cluster - they may not be deployed yet" - echo " Configuration changes will be applied when the pods start" - fi - else - echo " โŒ Could not get kubeconfig for $cluster - skipping pod restart" - fi - done - - echo " โœ… Completed ramenddr-cluster-operator pod restarts on managed clusters" - -# ramen-hub-operator-config caCertificates (s3StoreProfiles) are owned by regional DR / Ramen charts โ€” not this precheck. - - # Restart Velero pods on managed clusters to pick up new CA certificates - echo "7c. Restarting Velero pods on managed clusters..." - - MANAGED_CLUSTERS=$(oc get managedclusters -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Processing cluster: $cluster" - - # Get kubeconfig for the cluster - KUBECONFIG_FILE="/tmp/${cluster}-kubeconfig.yaml" - if [[ ! -f "$KUBECONFIG_FILE" ]]; then - # Fetch kubeconfig if not already available - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "$KUBECONFIG_FILE" 2>/dev/null; then - echo " Fetched kubeconfig for $cluster" - else - echo " โŒ Could not get kubeconfig for $cluster - skipping Velero pod restart" - continue - fi - fi - - # Find Velero pods in openshift-adp namespace - VELERO_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-adp -l component=velero -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - - if [[ -n "$VELERO_PODS" ]]; then - echo " Found Velero pods: $VELERO_PODS" - - for pod in $VELERO_PODS; do - echo " Deleting pod $pod to trigger restart..." - oc --kubeconfig="$KUBECONFIG_FILE" delete pod "$pod" -n openshift-adp --ignore-not-found=true || { - echo " Warning: Could not delete pod $pod" - } - done - - # Wait for pods to be deleted - echo " Waiting for pods to be terminated..." - for pod in $VELERO_PODS; do - oc --kubeconfig="$KUBECONFIG_FILE" wait --for=delete pod/"$pod" -n openshift-adp --timeout=60s 2>/dev/null || true - done - - # Wait for new pods to be running - echo " Waiting for new Velero pods to be running..." - MAX_WAIT_ATTEMPTS=30 - WAIT_INTERVAL=10 - attempt=0 - - while [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; do - attempt=$((attempt + 1)) - - NEW_PODS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pods -n openshift-adp -l component=velero -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "") - ALL_RUNNING=true - - if [[ -n "$NEW_PODS" ]]; then - for pod in $NEW_PODS; do - POD_STATUS=$(oc --kubeconfig="$KUBECONFIG_FILE" get pod "$pod" -n openshift-adp -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") - - if [[ "$POD_STATUS" != "Running" ]]; then - ALL_RUNNING=false - break - fi - done - - if [[ "$ALL_RUNNING" == "true" ]]; then - echo " โœ… All Velero pods are running on $cluster: $NEW_PODS" - break - else - echo " โณ Waiting for pods to be running (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - else - echo " โณ Waiting for pods to appear (attempt $attempt/$MAX_WAIT_ATTEMPTS)" - fi - - if [[ $attempt -lt $MAX_WAIT_ATTEMPTS ]]; then - sleep $WAIT_INTERVAL - fi - done - - if [[ $attempt -ge $MAX_WAIT_ATTEMPTS ]]; then - echo " โš ๏ธ Warning: Velero pods did not become ready within expected time on $cluster" - echo " The pods may still be starting - new CA certificates will be applied when ready" - fi - else - echo " โš ๏ธ Warning: Velero pods not found on $cluster - they may not be deployed yet" - echo " New CA certificates will be applied when the pods start" - fi - done - - echo " โœ… Completed Velero pod restarts on managed clusters" - - echo "8. Distributing certificate data to managed clusters..." - DISTRIBUTION_ATTEMPTS=3 - DISTRIBUTION_SLEEP=10 - - for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Distributing to $cluster..." - - KUBECONFIG_FILE="" - if oc get secret -n "$cluster" -o name | grep -E "(admin-kubeconfig|kubeconfig)" | head -1 | xargs -I {} oc get {} -n "$cluster" -o jsonpath='{.data.kubeconfig}' | base64 -d > "$WORK_DIR/${cluster}-kubeconfig.yaml" 2>/dev/null; then - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - fi - - if [[ -n "$KUBECONFIG_FILE" && -f "$KUBECONFIG_FILE" ]]; then - distribution_success=false - for dist_attempt in $(seq 1 $DISTRIBUTION_ATTEMPTS); do - echo " Distribution attempt $dist_attempt/$DISTRIBUTION_ATTEMPTS for $cluster..." - - if oc --kubeconfig="$KUBECONFIG_FILE" create configmap cluster-proxy-ca-bundle \ - --from-file=ca-bundle.crt="$WORK_DIR/combined-ca-bundle.crt" \ - -n openshift-config \ - --dry-run=client -o yaml | oc --kubeconfig="$KUBECONFIG_FILE" apply -f -; then - - if oc --kubeconfig="$KUBECONFIG_FILE" patch proxy/cluster --type=merge --patch='{"spec":{"trustedCA":{"name":"cluster-proxy-ca-bundle"}}}'; then - echo " โœ… Certificate data distributed to $cluster (attempt $dist_attempt)" - distribution_success=true - break - else - echo " โš ๏ธ ConfigMap created but proxy update failed for $cluster (attempt $dist_attempt)" - fi - else - echo " โš ๏ธ ConfigMap creation failed for $cluster (attempt $dist_attempt)" - fi - - if [[ $dist_attempt -lt $DISTRIBUTION_ATTEMPTS ]]; then - echo " โณ Waiting $DISTRIBUTION_SLEEP seconds before retry..." - sleep $DISTRIBUTION_SLEEP - fi - done - - if [[ "$distribution_success" != "true" ]]; then - echo " โŒ Failed to distribute certificate data to $cluster after $DISTRIBUTION_ATTEMPTS attempts" - echo " This may cause DR prerequisites check to fail" - fi - else - echo " โŒ Could not get kubeconfig for $cluster - skipping distribution" - fi - done - - echo "9. Verifying certificate distribution to managed clusters..." - verification_failed=false - REQUIRED_VERIFICATION_CLUSTERS=("$PRIMARY_CLUSTER" "$SECONDARY_CLUSTER") - VERIFIED_CLUSTERS=() - - for cluster in $MANAGED_CLUSTERS; do - if [[ "$cluster" == "local-cluster" ]]; then - continue - fi - - echo " Verifying distribution to $cluster..." - KUBECONFIG_FILE="$WORK_DIR/${cluster}-kubeconfig.yaml" - - if [[ -f "$KUBECONFIG_FILE" ]]; then - configmap_exists=$(oc --kubeconfig="$KUBECONFIG_FILE" get configmap cluster-proxy-ca-bundle -n openshift-config &>/dev/null && echo "true" || echo "false") - configmap_size=$(oc --kubeconfig="$KUBECONFIG_FILE" get configmap cluster-proxy-ca-bundle -n openshift-config -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c || echo "0") - proxy_configured=$(oc --kubeconfig="$KUBECONFIG_FILE" get proxy cluster -o jsonpath='{.spec.trustedCA.name}' 2>/dev/null || echo "") - - if [[ "$configmap_exists" == "true" && $configmap_size -gt 100 && "$proxy_configured" == "cluster-proxy-ca-bundle" ]]; then - echo " โœ… $cluster: ConfigMap exists (${configmap_size} bytes), proxy configured" - VERIFIED_CLUSTERS+=("$cluster") - else - echo " โŒ $cluster: ConfigMap verification failed" - echo " ConfigMap exists: $configmap_exists" - echo " ConfigMap size: $configmap_size bytes" - echo " Proxy configured: $proxy_configured" - verification_failed=true - fi - else - echo " โŒ $cluster: No kubeconfig available for verification" - verification_failed=true - fi - done - - # Check if all required clusters are verified - echo "10. Validating verification results..." - MISSING_VERIFICATION_CLUSTERS=() - for required_cluster in "${REQUIRED_VERIFICATION_CLUSTERS[@]}"; do - if [[ " ${VERIFIED_CLUSTERS[@]} " =~ " ${required_cluster} " ]]; then - echo " โœ… $required_cluster: Certificate distribution verified" - else - echo " โŒ $required_cluster: Certificate distribution NOT verified" - MISSING_VERIFICATION_CLUSTERS+=("$required_cluster") - fi - done - - if [[ ${#MISSING_VERIFICATION_CLUSTERS[@]} -gt 0 ]]; then - echo "" - echo "โŒ CRITICAL ERROR: Certificate distribution verification failed for required clusters:" - for missing in "${MISSING_VERIFICATION_CLUSTERS[@]}"; do - echo " - $missing" - done - echo "" - echo "The ODF SSL certificate extractor job requires successful certificate distribution" - echo "to ALL managed clusters ($PRIMARY_CLUSTER and $SECONDARY_CLUSTER)." - echo "" - echo "Without proper certificate distribution, the DR setup will fail." - echo "Please check cluster connectivity and kubeconfig availability." - echo "" - echo "Job will exit with error code 1." - exit 1 - fi - - if [[ "$verification_failed" == "true" ]]; then - echo "" - echo "โš ๏ธ Certificate distribution verification failed for some clusters" - echo " This may cause DR prerequisites check to fail" - echo " Manual intervention may be required" - echo "" - echo "Job will exit with error code 1." - exit 1 - else - echo "" - echo "โœ… All managed clusters verified successfully" - fi - - echo "" - echo "โœ… ODF SSL certificate management completed successfully!" - echo " - Hub cluster CA bundle: Updated (includes trusted CA + ingress CA)" - echo " - Hub cluster proxy: Configured" - echo " - Managed clusters: ramenddr-cluster-operator pods restarted" - echo " - Managed clusters: Velero pods restarted (openshift-adp namespace)" - echo " - Managed clusters: Certificate data distributed (includes ingress CAs)" - echo "" - echo "This follows Red Hat ODF Disaster Recovery certificate management guidelines" - echo "for secure SSL access across clusters in the regional DR setup." - echo "Ramen hub s3StoreProfiles / caCertificates are configured by regional DR or Ramen charts, not this job." - } - - # Execute main function with retry logic - while true; do - if main_execution; then - echo "๐ŸŽ‰ Certificate extraction completed successfully!" - exit 0 - else - if [[ $RETRY_COUNT -lt $MAX_RETRIES ]]; then - echo "๐Ÿ”„ Main execution failed, retrying..." - exponential_backoff - continue - else - echo "๐Ÿ’ฅ Max retries exceeded. Job will exit but ArgoCD can retry the sync." - echo " This is a temporary failure - the job will be retried on next ArgoCD sync." - exit 1 - fi - fi - done -EOF - - echo "Certificate extraction job created" - - echo "Waiting for certificate extraction to complete..." - attempt=0 - while [[ $attempt -lt $MAX_ATTEMPTS ]]; do - attempt=$((attempt + 1)) - echo " Attempt $attempt/$MAX_ATTEMPTS" - - if oc wait --for=condition=complete job/odf-ssl-certificate-extractor -n "$CLUSTER_CA_MGT_NAMESPACE" --timeout=60s 2>/dev/null; then - echo " โœ… Certificate extraction completed successfully" - return 0 - else - echo " โณ Certificate extraction still running, waiting..." - sleep $SLEEP_INTERVAL - fi - done - - echo " โŒ Certificate extraction did not complete within expected time" - return 1 -} - -# Main execution with retry logic -main_execution() { - echo "๐Ÿ” Starting certificate distribution check with retry logic..." - - # First, wait for required clusters to be ready - echo "โณ Waiting for required clusters to be available and joined before proceeding..." - if wait_for_cluster_readiness; then - echo "โœ… All required clusters are ready - proceeding with certificate checks" - else - echo "โš ๏ธ Some clusters are not ready yet, but continuing anyway..." - echo " The certificate extraction will be attempted when clusters become ready" - fi - - attempt=1 - while [[ $attempt -le $MAX_ATTEMPTS ]]; do - echo "=== Certificate Distribution Attempt $attempt/$MAX_ATTEMPTS ===" - - if check_certificate_distribution; then - echo "โœ… Certificate distribution is complete and verified" - echo " All clusters have proper CA bundles" - echo "๐ŸŽฏ ODF SSL certificate precheck completed successfully" - echo " Ready for DR prerequisites check" - exit 0 - else - echo "โŒ Certificate distribution is incomplete or missing" - - echo "๐Ÿงน Cleaning up placeholder ConfigMaps..." - cleanup_placeholder_configmaps - - echo " Triggering certificate extraction (attempt $attempt/$MAX_ATTEMPTS)..." - - # shellcheck disable=SC2119 - if trigger_certificate_extraction; then - echo "โœ… Certificate extraction completed successfully" - echo " Re-verifying distribution..." - - sleep 10 - - if check_certificate_distribution; then - echo "โœ… Certificate distribution verified after extraction" - echo "๐ŸŽฏ ODF SSL certificate precheck completed successfully" - echo " Ready for DR prerequisites check" - exit 0 - else - echo "โš ๏ธ Certificate extraction completed but distribution still incomplete" - echo " Will retry in $SLEEP_INTERVAL seconds..." - fi - else - echo "โŒ Certificate extraction failed (attempt $attempt/$MAX_ATTEMPTS)" - echo " Will retry in $SLEEP_INTERVAL seconds..." - fi - fi - - if [[ $attempt -lt $MAX_ATTEMPTS ]]; then - echo "โณ Waiting $SLEEP_INTERVAL seconds before next attempt..." - sleep $SLEEP_INTERVAL - fi - - ((attempt++)) - done - - echo "โŒ Certificate distribution failed after $MAX_ATTEMPTS attempts" - echo " This may affect DR prerequisites check" - echo " Manual intervention may be required" - exit 1 -} - -# Call main execution -main_execution diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index f62c694..795e16c 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -1,4 +1,4 @@ -{{/* Primary cluster name for use in jobs, MirrorPeer, DRPC, etc. */}} +{{/* Primary cluster name for use in MirrorPeer, jobs, etc. */}} {{- define "rdr.primaryClusterName" -}} {{- $dr := index .Values.regionalDR 0 -}} {{- $override := index (index (.Values.clusterOverrides | default dict) "primary" | default dict) "name" -}} @@ -14,56 +14,24 @@ {{- $override | default $fallback -}} {{- end -}} -{{/* regionalDR[0].name (ClusterSet); Submariner broker namespace = name + "-broker" */}} +{{/* regionalDR[0].name (ClusterSet) */}} {{- define "rdr.regionalDRClusterSetName" -}} {{- $dr := index .Values.regionalDR 0 -}} {{- $dr.name -}} {{- end -}} -{{- define "rdr.submarinerBrokerNamespace" -}} -{{ include "rdr.regionalDRClusterSetName" . }}-broker -{{- end -}} - -{{/* global.clusterPlatform (e.g. AWS, BareMetal): AWS gates AWS-only chart pieces. Case-insensitive; default AWS. */}} -{{- define "rdr.clusterPlatformAws" -}} -{{- $g := .Values.global | default dict -}} -{{- if eq "aws" (lower ($g.clusterPlatform | default "AWS" | toString)) -}}1{{- else -}}0{{- end -}} -{{- end -}} - -{{/* Submariner EC2 SG tagger job + RBAC: AWS platform and submariner.sgTagJobEnabled true. */}} -{{- define "rdr.submarinerSgTagJobEnabled" -}} -{{- $sm := .Values.submariner | default dict -}} -{{- $aws := eq "1" (include "rdr.clusterPlatformAws" . | trim) -}} -{{- $want := and (hasKey $sm "sgTagJobEnabled") (index $sm "sgTagJobEnabled") -}} -{{- if and $aws $want -}}1{{- else -}}0{{- end -}} -{{- end -}} - -{{/* ODF post-install fixes: prerequisites checker + Ramen trusted CA jobs/RBAC. Default on if .Values.odf.postInstallFixesEnabled omitted. */}} +{{/* ODF post-install fixes: MirrorPeer + prerequisites. Default on if omitted. */}} {{- define "rdr.odfPostInstallFixesEnabled" -}} {{- $odf := .Values.odf | default dict -}} {{- if not (hasKey $odf "postInstallFixesEnabled") -}}1{{- else if index $odf "postInstallFixesEnabled" -}}1{{- else -}}0{{- end -}} {{- end -}} -{{/* Ramen hub trusted-CA job: patch s3StoreProfiles from cluster-proxy-ca-bundle. Default on when enabled omitted. */}} -{{- define "rdr.odfRamenTrustedCaEnabled" -}} -{{- if ne "1" (include "rdr.odfPostInstallFixesEnabled" . | trim) -}}0{{- else -}} -{{- $cfg := .Values.odfRamenTrustedCa | default dict -}} -{{- if not (hasKey $cfg "enabled") -}}1{{- else if index $cfg "enabled" -}}1{{- else -}}0{{- end -}} -{{- end -}} -{{- end -}} - -{{/* ODF SSL extraction jobs, ACM CA policies, and spoke distribution. Default on when enabled omitted. */}} -{{- define "rdr.odfSslCertificateExtractorEnabled" -}} -{{- $cfg := .Values.odfSslCertificateExtractor | default dict -}} -{{- if not (hasKey $cfg "enabled") -}}1{{- else if index $cfg "enabled" -}}1{{- else -}}0{{- end -}} -{{- end -}} - -{{/* Namespace for ODF CA post-install Jobs. */}} +{{/* Namespace for ODF post-install Jobs. */}} {{- define "rdr.clusterCaMgtNamespace" -}} {{- .Values.clusterCaMgt.namespace | default "cluster-ca-mgt" -}} {{- end -}} -{{/* Stable checksum of packaged ansible/ (excludes dotfiles). Drives CM + Job drift on chart updates. */}} +{{/* Stable checksum of packaged ansible/ (excludes dotfiles). */}} {{- define "rdr.ansibleConfigChecksum" -}} {{- $paths := list -}} {{- range $path, $_ := .Files.Glob "ansible/**" -}} @@ -78,21 +46,10 @@ {{- $buf | sha256sum -}} {{- end -}} -{{/* Argo CD sync-options for the ansible ConfigMap. */}} {{- define "rdr.ansibleConfigMapArgoSyncOptions" -}} {{- .Values.ansible.configMapArgoSyncOptions | default "Prune=false,ServerSideApply=true" -}} {{- end -}} -{{/* Pod template annotation: keep ansible Jobs in sync with odf-dr-ansible content. */}} {{- define "rdr.ansibleJobPodAnnotations" -}} checksum/odf-dr-ansible: {{ include "rdr.ansibleConfigChecksum" . | quote }} {{- end -}} - -{{/* - opp.* aliases โ€” used by the SSL certificate templates which were originally in odf-opp. - They delegate to the canonical rdr.* helpers so there is a single implementation. -*/}} -{{- define "opp.primaryClusterName" -}}{{ include "rdr.primaryClusterName" . }}{{- end -}} -{{- define "opp.secondaryClusterName" -}}{{ include "rdr.secondaryClusterName" . }}{{- end -}} -{{- define "opp.clusterCaMgtNamespace" -}}{{ include "rdr.clusterCaMgtNamespace" . }}{{- end -}} -{{- define "opp.odfSslCertificateExtractorEnabled" -}}{{ include "rdr.odfSslCertificateExtractorEnabled" . }}{{- end -}} diff --git a/templates/configmap-odf-ssl-playbooks.yaml b/templates/configmap-odf-ssl-playbooks.yaml deleted file mode 100644 index ac96c95..0000000 --- a/templates/configmap-odf-ssl-playbooks.yaml +++ /dev/null @@ -1,37 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: odf-ssl-extractor-playbooks - namespace: {{ include "opp.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ssl-certificate-management -data: - odf-ssl-certificate-extraction.yml: | -{{ .Files.Get "scripts/ansible/odf-ssl-certificate-extraction.yml" | indent 4 }} - tasks_kubeconfig.yml: | -{{ .Files.Get "scripts/ansible/tasks/kubeconfig.yml" | indent 4 }} - tasks_kubeconfig_attempt.yml: | -{{ .Files.Get "scripts/ansible/tasks/kubeconfig-attempt.yml" | indent 4 }} - tasks_odf_ssl_extract_ca.yml: | -{{ .Files.Get "scripts/ansible/tasks/odf-ssl-extract-ca.yml" | indent 4 }} - tasks_odf_ssl_delete_pods.yml: | -{{ .Files.Get "scripts/ansible/tasks/odf-ssl-delete-pods.yml" | indent 4 }} - tasks_wait_dr_managedclusters_available.yml: | -{{ .Files.Get "scripts/ansible/tasks/wait-dr-managedclusters-available.yml" | indent 4 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: odf-ssl-precheck-playbooks - namespace: {{ include "opp.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ssl-certificate-management -data: - odf-ssl-precheck.yml: | -{{ .Files.Get "scripts/ansible/odf-ssl-precheck.yml" | indent 4 }} - tasks_kubeconfig.yml: | -{{ .Files.Get "scripts/ansible/tasks/kubeconfig.yml" | indent 4 }} - tasks_kubeconfig_attempt.yml: | -{{ .Files.Get "scripts/ansible/tasks/kubeconfig-attempt.yml" | indent 4 }} -{{- end }} diff --git a/templates/job-odf-dr-prerequisites.yaml b/templates/job-odf-dr-prerequisites.yaml index 51f71ca..372dbad 100644 --- a/templates/job-odf-dr-prerequisites.yaml +++ b/templates/job-odf-dr-prerequisites.yaml @@ -27,13 +27,17 @@ spec: - name: SECONDARY_CLUSTER value: {{ include "rdr.secondaryClusterName" . | quote }} - name: CA_MATERIAL_MODE - value: {{ (.Values.odfDrPrerequisites | default dict).caMaterialMode | default "legacy" | quote }} + value: {{ (.Values.odfDrPrerequisites | default dict).caMaterialMode | default "trust-bundle" | quote }} + - name: CA_BUNDLE_NAME + value: {{ (.Values.odfDrPrerequisites | default dict).caBundleName | default "vp-pattern-proxy-ca-bundle" | quote }} + - name: CA_BUNDLE_NAMESPACE + value: {{ (.Values.odfDrPrerequisites | default dict).caBundleNamespace | default "openshift-config" | quote }} command: - /bin/bash - -c - | {{ include "rdr.ansibleStageOnly" . | indent 10 }} - export PRIMARY_CLUSTER SECONDARY_CLUSTER CA_MATERIAL_MODE + export PRIMARY_CLUSTER SECONDARY_CLUSTER CA_MATERIAL_MODE CA_BUNDLE_NAME CA_BUNDLE_NAMESPACE ODF_SCRIPT="$STAGE/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh" echo "Running ODF prerequisites script with line-buffered output (kubectl logs -f will show progress)." if command -v stdbuf >/dev/null 2>&1; then diff --git a/templates/job-odf-ramen-trusted-ca.yaml b/templates/job-odf-ramen-trusted-ca.yaml deleted file mode 100644 index f751dc9..0000000 --- a/templates/job-odf-ramen-trusted-ca.yaml +++ /dev/null @@ -1,66 +0,0 @@ -{{- if eq "1" (include "rdr.odfRamenTrustedCaEnabled" . | trim) }} -{{- $t := .Values.odfRamenTrustedCa | default dict }} -apiVersion: batch/v1 -kind: Job -metadata: - name: odf-ramen-trusted-ca - namespace: {{ include "rdr.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ramen-trusted-ca - app.kubernetes.io/component: ramen-hub-ca - annotations: - # After MirrorPeer + DRPolicy (wave 8); before DRPC โ€” requires s3StoreProfiles and cluster-proxy-ca-bundle - argocd.argoproj.io/sync-wave: "9" - argocd.argoproj.io/sync-options: Prune=false - argocd.argoproj.io/compare-options: IgnoreExtraneous -spec: - backoffLimit: 5 - activeDeadlineSeconds: 14400 - template: - metadata: - annotations: - {{- include "rdr.ansibleJobPodAnnotations" . | nindent 8 }} - spec: - restartPolicy: Never - serviceAccountName: odf-ramen-trusted-ca - containers: - - name: odf-ramen-trusted-ca - image: {{ $.Values.ansible.containerImage }} - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" - env: - - name: PRIMARY_CLUSTER - value: {{ include "rdr.primaryClusterName" . | quote }} - - name: SECONDARY_CLUSTER - value: {{ include "rdr.secondaryClusterName" . | quote }} - - name: RAMEN_CM_WAIT_SECONDS - value: {{ $t.ramenS3WaitSeconds | default 3600 | quote }} - - name: TRUSTED_CA_WAIT_SECONDS - value: {{ $t.trustedCaWaitSeconds | default 3600 | quote }} - - name: POLL_INTERVAL - value: {{ $t.pollInterval | default 15 | quote }} - - name: KUBECONFIG - value: "" - command: - - /bin/bash - - -c - - | -{{ include "rdr.ansibleBootstrap" . | indent 10 }} - ansible-playbook -i inventory/hosts playbooks/odf_ramen_trusted_ca.yml \ - -e "odf_ramen_s3_wait_seconds=${RAMEN_CM_WAIT_SECONDS:-3600}" \ - -e "odf_ramen_trusted_ca_wait_seconds=${TRUSTED_CA_WAIT_SECONDS:-3600}" \ - -e "odf_ramen_poll_interval=${POLL_INTERVAL:-15}" - volumeMounts: - - name: ansible - mountPath: /ansible-cm - readOnly: true - volumes: - - name: ansible - configMap: - name: odf-dr-ansible -{{- end }} diff --git a/templates/job-odf-ssl-certificate-extraction.yaml b/templates/job-odf-ssl-certificate-extraction.yaml deleted file mode 100644 index 599613d..0000000 --- a/templates/job-odf-ssl-certificate-extraction.yaml +++ /dev/null @@ -1,75 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: batch/v1 -kind: Job -metadata: - name: odf-ssl-certificate-extractor - namespace: {{ include "opp.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-extraction - annotations: - argocd.argoproj.io/sync-wave: "1" - argocd.argoproj.io/sync-options: Prune=false - argocd.argoproj.io/compare-options: IgnoreExtraneous -spec: - template: - spec: - containers: - - name: odf-ssl-extractor - image: {{ $.Values.ansible.containerImage }} - workingDir: /playbooks - resources: - requests: - memory: "768Mi" - cpu: "200m" - limits: - memory: "1536Mi" - cpu: "1000m" - env: - - name: PRIMARY_CLUSTER - value: {{ include "opp.primaryClusterName" . | quote }} - - name: SECONDARY_CLUSTER - value: {{ include "opp.secondaryClusterName" . | quote }} - - name: ANSIBLE_NOCOLOR - value: "true" - - name: CLUSTER_READINESS_MAX_ATTEMPTS - value: {{ ((.Values.odfSslCertificateExtractor | default dict).clusterReadinessMaxAttempts | default 150) | quote }} - - name: CLUSTER_READINESS_SLEEP_SECONDS - value: {{ ((.Values.odfSslCertificateExtractor | default dict).clusterReadinessSleepSeconds | default 30) | quote }} - - name: CLUSTER_CA_MGT_NAMESPACE - value: {{ include "opp.clusterCaMgtNamespace" . | quote }} - command: - - ansible-playbook - args: - - -i - - localhost, - - -c - - local - - -v - - /playbooks/odf-ssl-certificate-extraction.yml - volumeMounts: - - name: playbooks - mountPath: /playbooks - readOnly: true - volumes: - - name: playbooks - configMap: - name: odf-ssl-extractor-playbooks - items: - - key: odf-ssl-certificate-extraction.yml - path: odf-ssl-certificate-extraction.yml - - key: tasks_kubeconfig.yml - path: tasks/kubeconfig.yml - - key: tasks_kubeconfig_attempt.yml - path: tasks/kubeconfig-attempt.yml - - key: tasks_odf_ssl_extract_ca.yml - path: tasks/odf-ssl-extract-ca.yml - - key: tasks_odf_ssl_delete_pods.yml - path: tasks/odf-ssl-delete-pods.yml - - key: tasks_wait_dr_managedclusters_available.yml - path: tasks/wait-dr-managedclusters-available.yml - restartPolicy: Never - serviceAccountName: odf-ssl-extractor-sa - backoffLimit: 10 - activeDeadlineSeconds: 7200 -{{- end }} diff --git a/templates/job-odf-ssl-certificate-precheck.yaml b/templates/job-odf-ssl-certificate-precheck.yaml deleted file mode 100644 index ff8994b..0000000 --- a/templates/job-odf-ssl-certificate-precheck.yaml +++ /dev/null @@ -1,66 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: batch/v1 -kind: Job -metadata: - name: odf-ssl-certificate-precheck - namespace: {{ include "opp.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-precheck - annotations: - argocd.argoproj.io/sync-wave: "2" - argocd.argoproj.io/sync-options: Prune=false - argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/hook: "PostSync" - argocd.argoproj.io/hook-delete-policy: "BeforeHookCreation" -spec: - # Each failed Pod (e.g. OOM) counts toward backoff; lower limit avoids many dead Pods while debugging. - backoffLimit: 5 - activeDeadlineSeconds: 7200 - template: - spec: - serviceAccountName: odf-ssl-certificate-precheck - restartPolicy: Never - containers: - - name: odf-ssl-precheck - image: {{ $.Values.ansible.containerImage }} - workingDir: /playbooks - # Ansible + kubernetes client; raise if OOMKilled (exit 137) during long until loops. - resources: - requests: - memory: "384Mi" - cpu: "100m" - limits: - memory: "768Mi" - cpu: "500m" - env: - - name: PRIMARY_CLUSTER - value: {{ include "opp.primaryClusterName" . | quote }} - - name: SECONDARY_CLUSTER - value: {{ include "opp.secondaryClusterName" . | quote }} - - name: CLUSTER_CA_MGT_NAMESPACE - value: {{ include "opp.clusterCaMgtNamespace" . | quote }} - command: - - ansible-playbook - args: - - -i - - localhost, - - -c - - local - - /playbooks/odf-ssl-precheck.yml - volumeMounts: - - name: playbooks - mountPath: /playbooks - readOnly: true - volumes: - - name: playbooks - configMap: - name: odf-ssl-precheck-playbooks - items: - - key: odf-ssl-precheck.yml - path: odf-ssl-precheck.yml - - key: tasks_kubeconfig.yml - path: tasks/kubeconfig.yml - - key: tasks_kubeconfig_attempt.yml - path: tasks/kubeconfig-attempt.yml -{{- end }} diff --git a/templates/job-submariner-prerequisites.yaml b/templates/job-submariner-prerequisites.yaml deleted file mode 100644 index 0d60e91..0000000 --- a/templates/job-submariner-prerequisites.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: submariner-prerequisites-checker - namespace: open-cluster-management - labels: - app.kubernetes.io/name: submariner-prerequisites - app.kubernetes.io/component: health-check - annotations: - argocd.argoproj.io/sync-wave: "7" -spec: - backoffLimit: 50 - activeDeadlineSeconds: 14400 - template: - metadata: - annotations: - {{- include "rdr.ansibleJobPodAnnotations" . | nindent 8 }} - spec: - restartPolicy: OnFailure - containers: - - name: prerequisites-checker - image: {{ $.Values.ansible.containerImage }} - env: - - name: PRIMARY_CLUSTER - value: {{ include "rdr.primaryClusterName" . | quote }} - - name: SECONDARY_CLUSTER - value: {{ include "rdr.secondaryClusterName" . | quote }} - - name: SUBMARINER_BROKER_NAMESPACE - value: {{ include "rdr.submarinerBrokerNamespace" . | quote }} - command: - - /bin/bash - - -c - - | -{{ include "rdr.ansibleStageOnly" . | indent 10 }} - export PRIMARY_CLUSTER SECONDARY_CLUSTER SUBMARINER_BROKER_NAMESPACE - SUB_SCRIPT="$STAGE/roles/submariner_prerequisites/files/submariner-prerequisites-check.sh" - echo "Running Submariner prerequisites script with line-buffered output (kubectl logs -f will show progress)." - if command -v stdbuf >/dev/null 2>&1; then - exec stdbuf -oL -eL bash "$SUB_SCRIPT" - else - exec bash "$SUB_SCRIPT" - fi - volumeMounts: - - name: ansible - mountPath: /ansible-cm - readOnly: true - volumes: - - name: ansible - configMap: - name: odf-dr-ansible - serviceAccountName: submariner-prerequisites-checker diff --git a/templates/job-submariner-sg-tag.yaml b/templates/job-submariner-sg-tag.yaml deleted file mode 100644 index 83f089b..0000000 --- a/templates/job-submariner-sg-tag.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{- if eq "1" (include "rdr.submarinerSgTagJobEnabled" . | trim) }} -apiVersion: batch/v1 -kind: Job -metadata: - name: submariner-sg-tagger - namespace: open-cluster-management - labels: - app.kubernetes.io/name: submariner-sg-tag - app.kubernetes.io/component: aws-tagging - annotations: - argocd.argoproj.io/sync-wave: "8" -spec: - backoffLimit: 3 - activeDeadlineSeconds: 3600 - template: - metadata: - annotations: - {{- include "rdr.ansibleJobPodAnnotations" . | nindent 8 }} - spec: - restartPolicy: OnFailure - containers: - - name: sg-tagger - image: {{ $.Values.ansible.containerImage }} - env: - - name: AWS_SDK_LOAD_CONFIG - value: "false" - - name: AWS_EC2_METADATA_DISABLED - value: "true" - command: - - /bin/bash - - -c - - | -{{ include "rdr.ansibleBootstrap" . | indent 10 }} - ansible-playbook -i inventory/hosts playbooks/submariner_sg_tag.yml - volumeMounts: - - name: ansible - mountPath: /ansible-cm - readOnly: true - volumes: - - name: ansible - configMap: - name: odf-dr-ansible - serviceAccountName: submariner-sg-tagger -{{- end }} diff --git a/templates/ocm-observability-bucket-claim.yaml b/templates/ocm-observability-bucket-claim.yaml new file mode 100644 index 0000000..586e75e --- /dev/null +++ b/templates/ocm-observability-bucket-claim.yaml @@ -0,0 +1,13 @@ +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: obc-observability + namespace: openshift-storage + labels: + app.kubernetes.io/name: observability + app.kubernetes.io/component: storage + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + generateBucketName: obc-observability-bucket + storageClassName: openshift-storage.noobaa.io diff --git a/templates/placement-binding-odf-ssl.yaml b/templates/placement-binding-observability-hub.yaml similarity index 57% rename from templates/placement-binding-odf-ssl.yaml rename to templates/placement-binding-observability-hub.yaml index 6d956e1..bce8914 100644 --- a/templates/placement-binding-odf-ssl.yaml +++ b/templates/placement-binding-observability-hub.yaml @@ -1,21 +1,19 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: policy.open-cluster-management.io/v1 kind: PlacementBinding metadata: - name: binding-odf-ssl-certificates + name: binding-observability-storage-hub namespace: open-cluster-management labels: - app.kubernetes.io/name: odf-ssl-certificate-management + app.kubernetes.io/name: observability app.kubernetes.io/component: placement-binding annotations: argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/sync-wave: "4" + argocd.argoproj.io/sync-wave: "2" placementRef: - name: placement-odf-ssl-certificates + name: placement-observability-hub kind: PlacementRule apiGroup: apps.open-cluster-management.io subjects: -- name: policy-odf-managed-cluster-ssl +- name: policy-observability-storage kind: Policy apiGroup: policy.open-cluster-management.io -{{- end }} diff --git a/templates/placement-binding-odf-ssl-hub.yaml b/templates/placement-binding-odf-ssl-hub.yaml deleted file mode 100644 index 302307f..0000000 --- a/templates/placement-binding-odf-ssl-hub.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: policy.open-cluster-management.io/v1 -kind: PlacementBinding -metadata: - name: binding-odf-ssl-certificate-management-hub - namespace: open-cluster-management - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: placement-binding - annotations: - argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/sync-wave: "3" -placementRef: - name: placement-odf-ssl-hub - kind: PlacementRule - apiGroup: apps.open-cluster-management.io -subjects: -- name: policy-odf-ssl-certificate-management - kind: Policy - apiGroup: policy.open-cluster-management.io -{{- end }} diff --git a/templates/placement-odf-ssl-hub.yaml b/templates/placement-observability-hub.yaml similarity index 67% rename from templates/placement-odf-ssl-hub.yaml rename to templates/placement-observability-hub.yaml index 3914677..4d6cf50 100644 --- a/templates/placement-odf-ssl-hub.yaml +++ b/templates/placement-observability-hub.yaml @@ -1,15 +1,14 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} apiVersion: apps.open-cluster-management.io/v1 kind: PlacementRule metadata: - name: placement-odf-ssl-hub + name: placement-observability-hub namespace: open-cluster-management labels: - app.kubernetes.io/name: odf-ssl-certificate-management + app.kubernetes.io/name: observability app.kubernetes.io/component: placement annotations: argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/sync-wave: "3" + argocd.argoproj.io/sync-wave: "2" spec: clusterConditions: - type: ManagedClusterConditionAvailable @@ -20,4 +19,3 @@ spec: operator: In values: - local-cluster -{{- end }} diff --git a/templates/placement-odf-ssl-certificates.yaml b/templates/placement-odf-ssl-certificates.yaml deleted file mode 100644 index c3fede5..0000000 --- a/templates/placement-odf-ssl-certificates.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: apps.open-cluster-management.io/v1 -kind: PlacementRule -metadata: - name: placement-odf-ssl-certificates - namespace: open-cluster-management - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: placement - annotations: - argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/sync-wave: "4" -spec: - clusterConditions: - - type: ManagedClusterConditionAvailable - status: "True" - clusterSelector: - matchExpressions: - - key: cluster.open-cluster-management.io/clusterset - operator: Exists - - key: purpose - operator: In - values: ["regionalDR"] - - key: vendor - operator: In - values: ["OpenShift"] - - key: name - operator: NotIn - values: ["local-cluster"] -{{- end }} diff --git a/templates/policy-observability-storage.yaml b/templates/policy-observability-storage.yaml new file mode 100644 index 0000000..ed462c0 --- /dev/null +++ b/templates/policy-observability-storage.yaml @@ -0,0 +1,101 @@ +apiVersion: policy.open-cluster-management.io/v1 +kind: Policy +metadata: + annotations: + policy.open-cluster-management.io/categories: CA Assessment Authorization and + Monitoring + policy.open-cluster-management.io/controls: CA-7 Continuous Monitoring + policy.open-cluster-management.io/standards: NIST SP 800-53 + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false + argocd.argoproj.io/health-check-ignore: "true" + argocd.argoproj.io/hook: "PostSync" + argocd.argoproj.io/sync-wave: "2" + labels: + open-cluster-management.io/policy-set: openshift-plus + name: policy-observability-storage + namespace: open-cluster-management +spec: + disabled: false + policy-templates: + - objectDefinition: + apiVersion: policy.open-cluster-management.io/v1 + kind: ConfigurationPolicy + metadata: + name: policy-observability-storage + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + spec: + object-templates: + - complianceType: musthave + objectDefinition: + apiVersion: batch/v1 + kind: Job + metadata: + name: configure-observability-storage + namespace: open-cluster-management-observability + labels: + app.kubernetes.io/name: observability + app.kubernetes.io/component: storage-config + annotations: + argocd.argoproj.io/hook: "PostSync" + argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/hook-delete-policy: "HookSucceeded" + spec: + template: + spec: + containers: + - name: storage-configurator + image: registry.redhat.io/openshift4/ose-cli:latest + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "200m" + command: + - /bin/bash + - -c + - | + set -euo pipefail + + echo "Configuring observability storage..." + + # Wait for OBC to be ready + echo "Waiting for ObjectBucketClaim to be ready..." + oc wait --for=condition=Bound objectbucketclaim/obc-observability -n openshift-storage --timeout=300s || echo "OBC not ready, continuing..." + + # Get OBC details + OBC_NAME=$(oc get objectbucketclaim obc-observability -n openshift-storage -o jsonpath='{.metadata.name}' 2>/dev/null || echo "") + OBC_NAMESPACE=$(oc get objectbucketclaim obc-observability -n openshift-storage -o jsonpath='{.metadata.namespace}' 2>/dev/null || echo "") + + if [[ -n "$OBC_NAME" && -n "$OBC_NAMESPACE" ]]; then + echo "OBC found: $OBC_NAME in namespace $OBC_NAMESPACE" + + # Update MultiClusterObservability with storage config + echo "Updating MultiClusterObservability with storage configuration..." + oc patch multiclusterobservability observability -n open-cluster-management-observability --type=merge --patch='{ + "spec": { + "storageConfigObject": { + "metricObjectStorage": { + "name": "'$OBC_NAME'", + "namespace": "'$OBC_NAMESPACE'" + } + } + } + }' || echo "Failed to update MultiClusterObservability" + else + echo "OBC not found, skipping storage configuration" + fi + + echo "Observability storage configuration completed" + env: + - name: KUBECONFIG + value: "" + restartPolicy: Never + serviceAccountName: observability-controller-sa + backoffLimit: 1 + activeDeadlineSeconds: 300 + remediationAction: enforce + severity: medium diff --git a/templates/policy-odf-managed-cluster-ssl.yaml b/templates/policy-odf-managed-cluster-ssl.yaml deleted file mode 100644 index 7fd799a..0000000 --- a/templates/policy-odf-managed-cluster-ssl.yaml +++ /dev/null @@ -1,75 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: policy.open-cluster-management.io/v1 -kind: Policy -metadata: - annotations: - policy.open-cluster-management.io/categories: CA Assessment Authorization and - Monitoring - policy.open-cluster-management.io/controls: CA-7 Continuous Monitoring - policy.open-cluster-management.io/standards: NIST SP 800-53 - argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/sync-options: Prune=false - argocd.argoproj.io/health-check-ignore: "true" - argocd.argoproj.io/hook: "PostSync" - argocd.argoproj.io/sync-wave: "4" - labels: - app.kubernetes.io/name: odf-managed-cluster-ssl - app.kubernetes.io/component: policy - name: policy-odf-managed-cluster-ssl - namespace: open-cluster-management -spec: - disabled: true - policy-templates: - - objectDefinition: - apiVersion: policy.open-cluster-management.io/v1 - kind: ConfigurationPolicy - metadata: - name: policy-odf-managed-cluster-ssl - annotations: - argocd.argoproj.io/compare-options: IgnoreExtraneous - spec: - object-templates: - - complianceType: musthave - objectDefinition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: cluster-proxy-ca-bundle - namespace: openshift-config - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: ca-bundle - annotations: - policy.open-cluster-management.io/description: "CA bundle for ODF disaster recovery SSL access" - data: - ca-bundle.crt: | - # Placeholder for ODF SSL certificate bundle - # This will be populated by the certificate extraction job - # with actual certificates from hub and managed clusters - - complianceType: musthave - objectDefinition: - apiVersion: config.openshift.io/v1 - kind: Proxy - metadata: - name: cluster - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: proxy-config - spec: - trustedCA: - name: cluster-proxy-ca-bundle - - complianceType: musthave - objectDefinition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: odf-ssl-verification - namespace: openshift-storage - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: verification - annotations: - policy.open-cluster-management.io/description: "ODF SSL certificate verification status" - remediationAction: enforce - severity: high -{{- end }} diff --git a/templates/policy-odf-ssl-certificate-management.yaml b/templates/policy-odf-ssl-certificate-management.yaml deleted file mode 100644 index ab3c9e6..0000000 --- a/templates/policy-odf-ssl-certificate-management.yaml +++ /dev/null @@ -1,48 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: policy.open-cluster-management.io/v1 -kind: Policy -metadata: - annotations: - policy.open-cluster-management.io/categories: CA Assessment Authorization and - Monitoring - policy.open-cluster-management.io/controls: CA-7 Continuous Monitoring - policy.open-cluster-management.io/standards: NIST SP 800-53 - argocd.argoproj.io/compare-options: IgnoreExtraneous - argocd.argoproj.io/sync-options: Prune=false - argocd.argoproj.io/health-check-ignore: "true" - argocd.argoproj.io/hook: "PostSync" - argocd.argoproj.io/sync-wave: "3" - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: policy - name: policy-odf-ssl-certificate-management - namespace: open-cluster-management -spec: - disabled: false - policy-templates: - - objectDefinition: - apiVersion: policy.open-cluster-management.io/v1 - kind: ConfigurationPolicy - metadata: - name: policy-odf-ssl-certificate-management - annotations: - argocd.argoproj.io/compare-options: IgnoreExtraneous - spec: - object-templates: - # Do not enforce the ConfigMap body here. A metadata-only template reconciles to an empty - # cluster-proxy-ca-bundle and wipes ca-bundle.crt written by odf-ssl-certificate-extractor. - - complianceType: musthave - objectDefinition: - apiVersion: config.openshift.io/v1 - kind: Proxy - metadata: - name: cluster - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: proxy-config - spec: - trustedCA: - name: cluster-proxy-ca-bundle - remediationAction: enforce - severity: high -{{- end }} diff --git a/templates/rbac-odf-ramen-trusted-ca.yaml b/templates/rbac-odf-ramen-trusted-ca.yaml deleted file mode 100644 index 58bd331..0000000 --- a/templates/rbac-odf-ramen-trusted-ca.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if eq "1" (include "rdr.odfRamenTrustedCaEnabled" . | trim) }} -# Workloads run in clusterCaMgt.namespace; cluster-proxy-ca-bundle remains in openshift-config. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: odf-ramen-trusted-ca - namespace: {{ include "rdr.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ramen-trusted-ca - app.kubernetes.io/component: ramen-hub-ca - annotations: - argocd.argoproj.io/sync-wave: "9" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: odf-ramen-trusted-ca - labels: - app.kubernetes.io/name: odf-ramen-trusted-ca - app.kubernetes.io/component: ramen-hub-ca - annotations: - argocd.argoproj.io/sync-wave: "9" -rules: -# Hub: read cluster-proxy-ca-bundle; read/patch ramen-hub-operator-config -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "patch", "update", "create"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: odf-ramen-trusted-ca - labels: - app.kubernetes.io/name: odf-ramen-trusted-ca - app.kubernetes.io/component: ramen-hub-ca - annotations: - argocd.argoproj.io/sync-wave: "9" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: odf-ramen-trusted-ca -subjects: -- kind: ServiceAccount - name: odf-ramen-trusted-ca - namespace: {{ include "rdr.clusterCaMgtNamespace" . }} -{{- end }} diff --git a/templates/rbac-odf-ssl-certificate-precheck.yaml b/templates/rbac-odf-ssl-certificate-precheck.yaml deleted file mode 100644 index a16b7db..0000000 --- a/templates/rbac-odf-ssl-certificate-precheck.yaml +++ /dev/null @@ -1,62 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: odf-ssl-certificate-precheck - namespace: {{ include "opp.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-precheck - annotations: - argocd.argoproj.io/sync-wave: "2" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: odf-ssl-certificate-precheck - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-precheck - annotations: - argocd.argoproj.io/sync-wave: "2" -rules: -# Permissions for checking certificate distribution -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] -- apiGroups: ["config.openshift.io"] - resources: ["proxies"] - verbs: ["get", "patch"] -# Permissions for triggering certificate extraction -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get", "list", "create", "delete"] -# Permissions for accessing managed cluster secrets -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -# Permissions for managed cluster operations -- apiGroups: ["cluster.open-cluster-management.io"] - resources: ["managedclusters"] - verbs: ["get", "list"] -- apiGroups: ["internal.open-cluster-management.io"] - resources: ["managedclusterinfos"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: odf-ssl-certificate-precheck - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-precheck - annotations: - argocd.argoproj.io/sync-wave: "2" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: odf-ssl-certificate-precheck -subjects: -- kind: ServiceAccount - name: odf-ssl-certificate-precheck - namespace: {{ include "opp.clusterCaMgtNamespace" . }}{{- end }} diff --git a/templates/rbac-odf-ssl-extractor.yaml b/templates/rbac-odf-ssl-extractor.yaml deleted file mode 100644 index 02d0747..0000000 --- a/templates/rbac-odf-ssl-extractor.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- if eq "1" (include "rdr.odfSslCertificateExtractorEnabled" . | trim) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: odf-ssl-extractor-sa - namespace: {{ include "opp.clusterCaMgtNamespace" . }} - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-extraction - annotations: - argocd.argoproj.io/sync-wave: "1" ---- -# Hub-only permissions for the Job SA. Pushes to managed clusters use each cluster's imported kubeconfig -# (ACM admin-kubeconfig); those calls are authorized on the spoke, not by this ClusterRole. -# Workloads run in clusterCaMgt.namespace; cluster-proxy-ca-bundle remains in openshift-config. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: odf-ssl-extractor-role - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-extraction - annotations: - argocd.argoproj.io/sync-wave: "1" -rules: -- apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "list", "create", "update", "patch", "delete"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - resourceNames: ["router-ca"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "delete"] -- apiGroups: ["config.openshift.io"] - resources: ["proxies"] - verbs: ["get", "patch"] -- apiGroups: ["cluster.open-cluster-management.io"] - resources: ["managedclusters"] - verbs: ["get", "list"] -- apiGroups: ["internal.open-cluster-management.io"] - resources: ["managedclusterinfos"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: odf-ssl-extractor-rolebinding - labels: - app.kubernetes.io/name: odf-ssl-certificate-management - app.kubernetes.io/component: certificate-extraction - annotations: - argocd.argoproj.io/sync-wave: "1" -subjects: -- kind: ServiceAccount - name: odf-ssl-extractor-sa - namespace: {{ include "opp.clusterCaMgtNamespace" . }} -roleRef: - kind: ClusterRole - name: odf-ssl-extractor-role - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/templates/rbac-submariner-prerequisites.yaml b/templates/rbac-submariner-prerequisites.yaml deleted file mode 100644 index 423f90f..0000000 --- a/templates/rbac-submariner-prerequisites.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: submariner-prerequisites-checker - namespace: open-cluster-management - labels: - app.kubernetes.io/name: submariner-prerequisites - app.kubernetes.io/component: health-check - annotations: - argocd.argoproj.io/sync-wave: "7" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: submariner-prerequisites-checker - labels: - app.kubernetes.io/name: submariner-prerequisites - app.kubernetes.io/component: health-check - annotations: - argocd.argoproj.io/sync-wave: "7" -rules: -# Permissions for checking managed clusters -- apiGroups: ["cluster.open-cluster-management.io"] - resources: ["managedclusters"] - verbs: ["get", "list"] -- apiGroups: ["internal.open-cluster-management.io"] - resources: ["managedclusterinfos"] - verbs: ["get", "list"] -# Permissions for checking Submariner connectivity -- apiGroups: ["submariner.io"] - resources: ["submariners", "clusters"] - verbs: ["get", "list"] -# Permissions for accessing managed cluster secrets -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -# Permissions for checking pods and deployments -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] -- apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["get", "list"] -# Permissions for checking nodes -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list"] -# Permissions for checking CRDs -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: submariner-prerequisites-checker - labels: - app.kubernetes.io/name: submariner-prerequisites - app.kubernetes.io/component: health-check - annotations: - argocd.argoproj.io/sync-wave: "7" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: submariner-prerequisites-checker -subjects: -- kind: ServiceAccount - name: submariner-prerequisites-checker - namespace: open-cluster-management - diff --git a/templates/rbac-submariner-sg-tag.yaml b/templates/rbac-submariner-sg-tag.yaml deleted file mode 100644 index ef3d9be..0000000 --- a/templates/rbac-submariner-sg-tag.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{{- if eq "1" (include "rdr.submarinerSgTagJobEnabled" . | trim) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: submariner-sg-tagger - namespace: open-cluster-management - labels: - app.kubernetes.io/name: submariner-sg-tag - app.kubernetes.io/component: aws-tagging - annotations: - argocd.argoproj.io/sync-wave: "8" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: submariner-sg-tagger - labels: - app.kubernetes.io/name: submariner-sg-tag - app.kubernetes.io/component: aws-tagging - annotations: - argocd.argoproj.io/sync-wave: "8" -rules: -# Permissions for accessing managed clusters -- apiGroups: ["cluster.open-cluster-management.io"] - resources: ["managedclusters"] - verbs: ["get", "list"] -- apiGroups: ["internal.open-cluster-management.io"] - resources: ["managedclusterinfos"] - verbs: ["get", "list"] -# Permissions for accessing managed cluster secrets (kubeconfigs and AWS credentials) -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] -# Permissions for accessing infrastructure resources (to get infrastructure name and AWS region) -- apiGroups: ["config.openshift.io"] - resources: ["infrastructures"] - verbs: ["get", "list"] -# Permissions for accessing ClusterDeployment (to get AWS region) -- apiGroups: ["hive.openshift.io"] - resources: ["clusterdeployments"] - verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: submariner-sg-tagger - labels: - app.kubernetes.io/name: submariner-sg-tag - app.kubernetes.io/component: aws-tagging - annotations: - argocd.argoproj.io/sync-wave: "8" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: submariner-sg-tagger -subjects: -- kind: ServiceAccount - name: submariner-sg-tagger - namespace: open-cluster-management -{{- end }} diff --git a/templates/submariner_addon_install.yaml b/templates/submariner_addon_install.yaml deleted file mode 100644 index d40a170..0000000 --- a/templates/submariner_addon_install.yaml +++ /dev/null @@ -1,66 +0,0 @@ -{{- $dr := index .Values.regionalDR 0 }} -{{- $clusterSet := $dr.name }} -{{- $globalnetEnabled := $dr.globalnetEnabled }} ---- -apiVersion: v1 -kind: Namespace -metadata: - annotations: - argocd.argoproj.io/sync-wave: "3" - name: {{ $clusterSet }}-broker ---- -apiVersion: submariner.io/v1alpha1 -kind: Broker -metadata: - name: submariner-broker - namespace: {{ $clusterSet }}-broker - labels: - cluster.open-cluster-management.io/backup: submariner - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - globalnetEnabled: {{ $globalnetEnabled | default false }} - - -{{- $primaryName := include "rdr.primaryClusterName" . | trim }} -{{- $secondaryName := include "rdr.secondaryClusterName" . | trim }} -{{- range list (dict "name" $primaryName) (dict "name" $secondaryName) }} -{{- $cluster := . }} -{{- $awsPlat := eq "1" (include "rdr.clusterPlatformAws" $ | trim) }} ---- -apiVersion: addon.open-cluster-management.io/v1alpha1 -kind: ManagedClusterAddOn -metadata: - name: submariner - namespace: {{ $cluster.name }} - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - installNamespace: submariner-operator - ---- -apiVersion: submarineraddon.open-cluster-management.io/v1alpha1 -kind: SubmarinerConfig -metadata: - name: submariner - namespace: {{ $cluster.name }} - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - gatewayConfig: - gateways: 1 -{{- if $awsPlat }} - aws: - instanceType: {{ $.Values.submariner.instanceType }} -{{- end }} - {{- if eq $.Values.submariner.cableDriver "libreswan" }} - IPSecNATTPort: {{ $.Values.submariner.ipsecNatPort }} - NATTEnable: {{ $.Values.submariner.NATTEnable }} - {{- end }} - cableDriver: {{ $.Values.submariner.cableDriver }} -{{- if $awsPlat }} - credentialsSecret: - name: {{ $cluster.name }}-cluster-aws-creds -{{- end }} - -{{- end }} diff --git a/values.yaml b/values.yaml index 116d8a8..e5084cb 100644 --- a/values.yaml +++ b/values.yaml @@ -1,22 +1,16 @@ --- -# Namespace for ODF SSL / CA post-install Jobs and RBAC (not cluster-proxy-ca-bundle). +# Namespace for ODF post-install Jobs (prerequisites checker). clusterCaMgt: - # -- Namespace for ODF CA prerequisites and Ramen trusted-CA workloads. + # -- Namespace for ODF CA prerequisites workloads. namespace: cluster-ca-mgt # -- Create clusterCaMgt.namespace when installing the chart. createNamespace: false -global: - # -- Base domain shared by all clusters (e.g. example.com). Used to derive per-cluster baseDomain. - clusterDomain: cluster.example.com - # -- Cloud platform type. AWS enables Hive ExternalSecret, ClusterDeployment platform.aws, Submariner gateway/credentials and SG-tag job. Use non-AWS (e.g. BareMetal) to skip those. - clusterPlatform: AWS - -# ODF post-install automation: MirrorPeer, prerequisites checker + Ramen hub trusted CA jobs/RBAC. +# ODF post-install automation: MirrorPeer + prerequisites checker. odf: - # -- Enable ODF post-install automation (MirrorPeer, prerequisites checker, Ramen trusted-CA jobs/RBAC). + # -- Enable ODF post-install automation (MirrorPeer, prerequisites checker). postInstallFixesEnabled: true - # When postInstallFixesEnabled is false: chart creates DRCluster CRs (hub) for primary/secondary; set S3 profile names that exist in hub Ramen config. + # When postInstallFixesEnabled is false: reserved for DRCluster S3 profile names. drCluster: # -- S3 profile name for the primary DRCluster CR. Required only when postInstallFixesEnabled is false. primaryS3ProfileName: "" @@ -24,24 +18,26 @@ odf: secondaryS3ProfileName: "" # DR pair cluster names. Must match ACM ManagedCluster metadata.name. -# Override via extraValueFiles (e.g. overrides/values-cluster-names.yaml). regionalDR: - - # -- Name of this DR pair set. Must be unique within the regionalDR list and match the ACM policy placement label. + - # -- Name of this DR pair set. Must be unique within the regionalDR list. name: resilient - # -- Enable Submariner Globalnet. Required when primary and secondary cluster CIDRs overlap. - globalnetEnabled: false - -submariner: - # -- EC2 instance type for Submariner gateway nodes. - instanceType: m5.xlarge - # -- IPsec NAT-T UDP port used by Submariner. - ipsecNatPort: 4500 - # -- Enable NAT traversal (NAT-T) for Submariner IPsec tunnels. - NATTEnable: true - # -- Submariner cable driver (vxlan or libreswan). - cableDriver: vxlan - # -- Enable EC2 security group tagging job. AWS only; requires global.clusterPlatform=AWS. - sgTagJobEnabled: false + clusters: + primary: + # -- ACM ManagedCluster name for the primary site. + name: ocp-primary + secondary: + # -- ACM ManagedCluster name for the secondary site. + name: ocp-secondary + +# When odf.postInstallFixesEnabled is true: odf-dr-prerequisites job/RBAC. +odfDrPrerequisites: + # -- CA completeness check mode. Use trust-bundle with vp-manage-proxy-cluster-ca + # (validates PEM and identical CA ConfigMaps). legacy expects '# CA from โ€ฆ' markers. + caMaterialMode: trust-bundle + # -- CA ConfigMap name to validate (vp-manage-proxy-cluster-ca configMapName). + caBundleName: vp-pattern-proxy-ca-bundle + # -- Namespace of the CA ConfigMap. + caBundleNamespace: openshift-config ansible: # -- Container image used for Ansible post-install jobs. @@ -50,29 +46,3 @@ ansible: verbosity: 0 # -- Argo CD resource sync-options applied to the Ansible ConfigMap. configMapArgoSyncOptions: Prune=false,ServerSideApply=true - -# When odf.postInstallFixesEnabled is true: odf-dr-prerequisites + odf-ramen-trusted-ca jobs/RBAC below. -odfDrPrerequisites: - # -- CA completeness check mode. legacy requires '# CA from โ€ฆ' markers from SSL extraction; - # trust-bundle validates PEM content and identical cluster-proxy-ca-bundle across clusters (vp-manage-proxy-cluster-ca). - caMaterialMode: legacy - -# Post-sync job odf-ramen-trusted-ca (wave 9): CA from hub cluster-proxy-ca-bundle; waits for Ramen s3StoreProfiles. -odfRamenTrustedCa: - # -- When false, skip the job that patches ramen-hub-operator-config s3StoreProfiles. - enabled: true - # -- Maximum seconds to wait for Ramen s3StoreProfiles to be populated before the trusted-CA job gives up. - ramenS3WaitSeconds: 3600 - # -- Maximum seconds to wait for the hub cluster-proxy-ca-bundle trusted CA before the job gives up. - trustedCaWaitSeconds: 3600 - # -- Polling interval in seconds for readiness checks inside the trusted-CA job. - pollInterval: 15 - -# odf-ssl-certificate-extractor Job: wait for both DR ManagedClusters before extraction. -odfSslCertificateExtractor: - # -- When false, skip SSL extraction/precheck jobs and ACM CA policies (use vp-manage-proxy-cluster-ca-chart). - enabled: true - # -- Maximum number of attempts to check DR ManagedCluster readiness before the extractor job fails. - clusterReadinessMaxAttempts: 150 - # -- Seconds to sleep between each ManagedCluster readiness poll attempt. - clusterReadinessSleepSeconds: 30 From 71e56c71a86f225cf418e9b57d610b80f666dc8b Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 5 Aug 2026 12:59:27 -0500 Subject: [PATCH 3/4] Fix release typo and bump version --- Chart.yaml | 2 +- README.md | 6 +++--- README.md.gotmpl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index f13359a..0b827a7 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -7,4 +7,4 @@ keywords: - odf - pattern - regionaldr -version: 0.0.4 +version: 0.1.0 diff --git a/README.md b/README.md index f119f4e..bec87e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # odf-dr-chart -![Version: 0.0.4](https://img.shields.io/badge/Version-0.0.4-informational?style=flat-square) +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ODF-specific Regional DR chart: MirrorPeer, ODF prerequisites, and observability ObjectBucketClaim/storage policy. Submariner and s3-ssl live in opp-policy-chart. @@ -10,11 +10,11 @@ Ramen `s3StoreProfiles` `caCertificates` injection is owned by **opp-policy-char ## Notable changes -v0.0.4 - Keep only ODF/MirrorPeer/observability; move Submariner and SSL to opp-policy; drop ramen trusted-CA job +v0.1.0 - Keep only ODF/MirrorPeer/observability; move Submariner and SSL to opp-policy; drop ramen trusted-CA job v0.0.3 - Add ObjectBucketClaim and policy-observability-storage (from opp-policy-chart) -v0.1.0 - Initial release +v0.0.1 - Initial release ## Values diff --git a/README.md.gotmpl b/README.md.gotmpl index 29cc948..c2cdb1d 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -11,11 +11,11 @@ Ramen `s3StoreProfiles` `caCertificates` injection is owned by **opp-policy-char ## Notable changes -v0.0.4 - Keep only ODF/MirrorPeer/observability; move Submariner and SSL to opp-policy; drop ramen trusted-CA job +v0.1.0 - Keep only ODF/MirrorPeer/observability; move Submariner and SSL to opp-policy; drop ramen trusted-CA job v0.0.3 - Add ObjectBucketClaim and policy-observability-storage (from opp-policy-chart) -v0.1.0 - Initial release +v0.0.1 - Initial release {{ template "chart.homepageLine" . }} From e86748101a1aae1e2cf2fa6ae34056d7d438f75f Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Wed, 5 Aug 2026 13:04:51 -0500 Subject: [PATCH 4/4] Fix shellcheck findings in ODF DR prerequisites script. Restore the legacy primary CA marker check after a bad merge and quote CA bundle ConfigMap names. Co-authored-by: Cursor --- .../files/odf-dr-prerequisites-check.sh | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh b/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh index 770861a..691ffb1 100755 --- a/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh +++ b/ansible/roles/odf_dr_prerequisites/files/odf-dr-prerequisites-check.sh @@ -6,7 +6,7 @@ if [[ -z "${ODF_PREREQ_LINEBUF:-}" ]] && command -v stdbuf >/dev/null 2>&1; then fi set -euo pipefail -CA_BUNDLE_NAME="$CA_BUNDLE_NAME" +CA_BUNDLE_NAME="${CA_BUNDLE_NAME:-vp-pattern-proxy-ca-bundle}" CA_BUNDLE_NAMESPACE="${CA_BUNDLE_NAMESPACE:-openshift-config}" echo "Starting ODF DR prerequisites check..." @@ -256,13 +256,14 @@ check_ca_configuration() { echo "Checking CA configuration on $cluster..." # Check if $CA_BUNDLE_NAME ConfigMap exists - if ! oc --kubeconfig="$kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" &>/dev/null; then + if ! oc --kubeconfig="$kubeconfig" get configmap "$CA_BUNDLE_NAME" -n "$CA_BUNDLE_NAMESPACE" &>/dev/null; then report_check_failure "CA config ($cluster): ConfigMap $CA_BUNDLE_NAME not found in $CA_BUNDLE_NAMESPACE" return 1 fi # Check if ConfigMap has certificate data - local ca_bundle_size=$(oc --kubeconfig="$kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c || echo "0") + local ca_bundle_size + ca_bundle_size=$(oc --kubeconfig="$kubeconfig" get configmap "$CA_BUNDLE_NAME" -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null | wc -c || echo "0") ca_bundle_size=$(echo "$ca_bundle_size" | tr -d ' \n') if [[ $ca_bundle_size -lt 100 ]]; then report_check_failure "CA config ($cluster): $CA_BUNDLE_NAME data ca-bundle.crt too small or empty (bytes: $ca_bundle_size)" @@ -307,8 +308,10 @@ check_ca_material_legacy_markers() { return 1 fi - if [[ "$primary_cert_count" -lt "$MIN_CERTS" ]]; then - report_check_failure "CA material: primary ($PRIMARY_CLUSTER) bundle has only ${primary_cert_count} certificates (expected at least ${MIN_CERTS})" + if [[ "$primary_ca_bundle" != *"# CA from hub-ca"* ]]; then + echo "Available markers in primary CA bundle:" + echo "$primary_ca_bundle" | grep "^# CA from" || echo "No CA markers found" + report_check_failure "CA material: primary bundle missing marker '# CA from hub-ca'" return 1 fi @@ -393,9 +396,12 @@ check_ca_material_completeness() { echo "Checking CA material completeness across all clusters (mode: ${CA_MATERIAL_MODE})..." # Extract CA bundle from each cluster - local hub_ca_bundle=$(oc --kubeconfig="$hub_kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - local primary_ca_bundle=$(oc --kubeconfig="$primary_kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") - local secondary_ca_bundle=$(oc --kubeconfig="$secondary_kubeconfig" get configmap $CA_BUNDLE_NAME -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local hub_ca_bundle + hub_ca_bundle=$(oc --kubeconfig="$hub_kubeconfig" get configmap "$CA_BUNDLE_NAME" -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local primary_ca_bundle + primary_ca_bundle=$(oc --kubeconfig="$primary_kubeconfig" get configmap "$CA_BUNDLE_NAME" -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") + local secondary_ca_bundle + secondary_ca_bundle=$(oc --kubeconfig="$secondary_kubeconfig" get configmap "$CA_BUNDLE_NAME" -n "$CA_BUNDLE_NAMESPACE" -o jsonpath='{.data.ca-bundle\.crt}' 2>/dev/null || echo "") # Check if all CA bundles exist and have reasonable size if [[ -z "$hub_ca_bundle" || ${#hub_ca_bundle} -lt 100 ]]; then