From 8883f50b7b6a8b0311c15db28f33d277c59ddd3f Mon Sep 17 00:00:00 2001 From: Akos Eros Date: Thu, 19 Mar 2026 14:44:24 +0100 Subject: [PATCH] feat: Add option to use preexisting clusters for DR clusters Set byoc variable and add secrets to secret template --- Makefile | 10 + ansible/playbooks/validate_byoc.yml | 225 ++++++++++++++++++ .../rdr/templates/cluster_deployments.yaml | 87 +++++-- charts/hub/rdr/values.yaml | 14 ++ values-hub.yaml | 5 +- values-secret.yaml.template | 11 + 6 files changed, 336 insertions(+), 16 deletions(-) create mode 100644 ansible/playbooks/validate_byoc.yml diff --git a/Makefile b/Makefile index b6b36c3..735f646 100644 --- a/Makefile +++ b/Makefile @@ -2,4 +2,14 @@ # This Makefile includes the common pattern targets from Makefile-common # You can add custom targets above or below the include line +.PHONY: install +install: ramen-prereq pattern-install ## Installs the pattern onto a cluster (Loads secrets as well if configured) + + + include Makefile-common + +.PHONY: ramen-prereq +ramen-prereq: ## Check if values.byoc false do nothing, else run the precheck agains clusters accessed from values-secrets + echo "Running precheck for ramendr" + cd ansible && ansible-playbook -i hosts $(EXTRA_ARGS) $(EXTRA_VARS) playbooks/validate_byoc.yml diff --git a/ansible/playbooks/validate_byoc.yml b/ansible/playbooks/validate_byoc.yml new file mode 100644 index 0000000..444fd71 --- /dev/null +++ b/ansible/playbooks/validate_byoc.yml @@ -0,0 +1,225 @@ +--- +- name: Validates that the regional clusters are good to use (byoc) + hosts: localhost + connection: local + gather_facts: false + roles: + - role: rhvp.cluster_utils.pattern_settings + vars: + pattern_dir: "{{ playbook_dir }}/../../" + - role: rhvp.cluster_utils.find_vp_secrets + vars: + byoc_enabled: false + tasks: + - name: Set fact for clsutergroup name + ansible.builtin.set_fact: + clustergroup_name: "{{ values_global.main.clusterGroupName }}" + + - name: Load values-{clustergroup_name}.yaml + ansible.builtin.set_fact: + values_clustergroup: "{{ lookup('file', pattern_dir + '/values-' + clustergroup_name + '.yaml') | from_yaml }}" + + - name: Set fact for thew rdr app + ansible.builtin.set_fact: + rdr_app: "{{ values_clustergroup.clusterGroup.applications.rdr }}" + + - name: Load rdr values yaml + ansible.builtin.set_fact: + rdr_values: "{{ lookup('file', pattern_dir + rdr_app.path + '/values.yaml') | from_yaml }}" + + - name: Set byoc for the playbook + ansible.builtin.set_fact: + byoc_enabled: true + when: + - ((rdr_app.overrides | default([]) | selectattr('name', 'eq', 'byoc')|first).value | default(false)) or (rdr_values.byoc) + + - name: Set fact for secretStore backend + ansible.builtin.set_fact: + secrets_backing_store: "{{ values_global.global.secretStore.backend | default('vault') }}" + when: byoc_enabled + + - name: Parse secrets data + no_log: "{{ hide_sensitive_output | default(true) }}" + rhvp.cluster_utils.parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_data }}" + secrets_backing_store: "{{ secrets_backing_store }}" + register: secrets_results + when: byoc_enabled + + - name: Check byoc if we need to + when: byoc_enabled + block: + - name: Set facts for dr clusters backend + ansible.builtin.set_fact: + clusters: + primary_cluster: "{{ secrets_results.parsed_secrets['ocp-primary_cluster_kubeconfig'].fields.kubeconfig }}" + secondary_cluster: "{{ secrets_results.parsed_secrets['ocp-secondary_cluster_kubeconfig'].fields.kubeconfig }}" + + - name: Get a tempfile for the kubeconfigs + ansible.builtin.tempfile: + state: directory + register: kubeconfigs_tempfolder + + - name: Store pull secrets in tempfile + ansible.builtin.copy: + dest: "{{ kubeconfigs_tempfolder.path }}/{{ cluster.key }}" + content: "{{ cluster.value }}" + mode: "0644" + loop_control: + loop_var: cluster + label: "{{ cluster.key }}" + loop: "{{ clusters | dict2items }}" + + - name: Check that both DR clusters are reachable + kubernetes.core.k8s_cluster_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/{{ cluster.key }}" + register: api_status + loop_control: + loop_var: cluster + label: "{{ cluster.key }}" + loop: "{{ clusters | dict2items }}" + + - name: Hub cluster version + kubernetes.core.k8s_info: + api: config.openshift.io/v1 + kind: ClusterVersion + name: version + register: hub_clusterversion_raw + + - name: Primary cluster version + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/primary_cluster" + api: config.openshift.io/v1 + kind: ClusterVersion + name: version + register: primary_clusterversion_raw + + - name: Secondary cluster version + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/secondary_cluster" + api: config.openshift.io/v1 + kind: ClusterVersion + name: version + register: secondary_clusterversion_raw + + - name: Set fact for cluster minor.major versions + ansible.builtin.set_fact: + primary_cluster_version: "{{ primary_clusterversion_raw.resources[0].status.desired.version.split('.')[:2] | join('.') }}" + secondary_cluster_version: "{{ secondary_clusterversion_raw.resources[0].status.desired.version.split('.')[:2] | join('.') }}" + hub_cluster_version: "{{ hub_clusterversion_raw.resources[0].status.desired.version.split('.')[:2] | join('.') }}" + + - name: Validate BYOC DR Clusters have the same minor version between 4.18 and 4.20 + ansible.builtin.assert: + that: + - hub_cluster_version is ansible.builtin.version("4.18", ">=") + - hub_cluster_version is ansible.builtin.version("4.20", "<=") + - primary_cluster_version is ansible.builtin.version(hub_cluster_version) + - secondary_cluster_version is ansible.builtin.version(hub_cluster_version) + fail_msg: | + FATAL: BYOC DR Clusters have different versions: + Primary: {{ primary_cluster_version }}, Secondary: {{ secondary_cluster_version }} HUB: {{ hub_cluster_version }}). + Currently only same minor verion supported by ramenDR and odf. + + - name: Hub cluster network + kubernetes.core.k8s_info: + api: config.openshift.io/v1 + kind: Network + name: cluster + register: hub_clusternetwork_raw + + - name: Primary cluster network + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/primary_cluster" + api: config.openshift.io/v1 + kind: Network + name: cluster + register: primary_clusternetwork_raw + + - name: Secondary cluster network + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/secondary_cluster" + api: config.openshift.io/v1 + kind: Network + name: cluster + register: secondary_clusternetwork_raw + + - name: Set fact for cluster network vars + ansible.builtin.set_fact: + hub_cluster_cluster_cidr: "{{ hub_clusternetwork_raw.resources[0].spec.clusterNetwork[0].cidr }}" + hub_cluster_service_cidr: "{{ hub_clusternetwork_raw.resources[0].spec.serviceNetwork[0] }}" + primary_cluster_cluster_cidr: "{{ primary_clusternetwork_raw.resources[0].spec.clusterNetwork[0].cidr }}" + primary_cluster_service_cidr: "{{ primary_clusternetwork_raw.resources[0].spec.serviceNetwork[0] }}" + secondary_cluster_cluster_cidr: "{{ secondary_clusternetwork_raw.resources[0].spec.clusterNetwork[0].cidr }}" + secondary_cluster_service_cidr: "{{ secondary_clusternetwork_raw.resources[0].spec.serviceNetwork[0] }}" + + - name: Validate BYOC DR Clusters have non overlapping pod networks. + ansible.builtin.assert: + that: + - not (hub_cluster_cluster_cidr | ansible.utils.ipaddr(primary_cluster_cluster_cidr)) + - not (hub_cluster_cluster_cidr | ansible.utils.ipaddr(secondary_cluster_cluster_cidr)) + - not (primary_cluster_cluster_cidr | ansible.utils.ipaddr(secondary_cluster_cluster_cidr)) + fail_msg: | + FATAL: BYOC DR Clusters have overlapping pod networks : + Primary: {{ primary_cluster_cluster_cidr }}, Secondary: {{ secondary_cluster_cluster_cidr }} HUB: {{ hub_cluster_cluster_cidr }}). + + - name: Validate BYOC DR Clusters have non overlapping service networks. + ansible.builtin.assert: + that: + - not (hub_cluster_service_cidr | ansible.utils.ipaddr(primary_cluster_service_cidr)) + - not (hub_cluster_service_cidr | ansible.utils.ipaddr(secondary_cluster_service_cidr)) + - not (primary_cluster_service_cidr | ansible.utils.ipaddr(secondary_cluster_service_cidr)) + fail_msg: | + FATAL: BYOC DR Clusters have overlapping service networks: + Primary: {{ primary_cluster_service_cidr }}, Secondary: {{ secondary_cluster_service_cidr }} HUB: {{ hub_cluster_service_cidr }}). + + - name: Get primary cluster node info + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/primary_cluster" + kind: Node + label_selectors: + - node-role.kubernetes.io/worker + register: primary_nodes_raw + + - name: Build instance type list (primary) + set_fact: + primary_worker_instance_types: "{{ primary_worker_instance_types | default([]) + [ item.metadata.labels['node.kubernetes.io/instance-type'] ] }}" + loop: "{{ primary_nodes_raw.resources }}" + loop_control: + label: "{{ item.metadata.name }}" + when: item.metadata.labels is defined and + 'node.kubernetes.io/instance-type' in item.metadata.labels + + - name: Make instance types unique (primary-cluster) + set_fact: + primary_worker_instance_types: "{{ primary_worker_instance_types | unique }}" + + - name: Get secondary cluster node info + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfigs_tempfolder.path }}/secondary_cluster" + kind: Node + label_selectors: + - node-role.kubernetes.io/worker + register: secondary_nodes_raw + + - name: Build instance type list (secondary) + set_fact: + secondary_worker_instance_types: "{{ secondary_worker_instance_types | default([]) + [ item.metadata.labels['node.kubernetes.io/instance-type'] ] }}" + loop_control: + label: "{{ item.metadata.name }}" + loop: "{{ secondary_nodes_raw.resources }}" + when: item.metadata.labels is defined and + 'node.kubernetes.io/instance-type' in item.metadata.labels + + - name: Make instance types unique (primary-cluster) + set_fact: + secondary_worker_instance_types: "{{ secondary_worker_instance_types | unique }}" + + - name: Validate BYOC DR Clusters have metal node to run virtual machine workload + ansible.builtin.assert: + that: + - primary_worker_instance_types | select('search', 'metal') | list | length > 0 + - secondary_worker_instance_types | select('search', 'metal') | list | length > 0 + fail_msg: | + FATAL: BYOC DR Clusters have no metal nodes: + Primary: {{ primary_worker_instance_types }}). + Secondary: {{ primary_worker_instance_types }}). diff --git a/charts/hub/rdr/templates/cluster_deployments.yaml b/charts/hub/rdr/templates/cluster_deployments.yaml index f1221fb..7da9bd4 100644 --- a/charts/hub/rdr/templates/cluster_deployments.yaml +++ b/charts/hub/rdr/templates/cluster_deployments.yaml @@ -9,17 +9,40 @@ {{- $cluster := . }} {{- $baseDomainRaw := index $cluster.install_config "baseDomain" }} {{- $baseDomainStr := default $defaultBaseDomain (and (kindIs "string" $baseDomainRaw) $baseDomainRaw) }} + +{{- if $.Values.byoc }} --- -apiVersion: v1 -kind: Namespace +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret metadata: - name: {{ $cluster.name }} - + name: {{ $cluster.name }}-auto-impport-kubeconfig + namespace: {{ $cluster.name }} + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + dataFrom: + - extract: + conversionStrategy: Default + decodingStrategy: None + key: secret/hub/{{ $cluster.name }}_cluster_kubeconfig + metadataPolicy: None + refreshInterval: 24h0m0s + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + creationPolicy: Owner + deletionPolicy: Retain + name: auto-import-secret + template: + engineVersion: v2 + mergePolicy: Replace + type: Opaque --- apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: {{ $cluster.name }}-cluster-private-key + name: {{ $cluster.name }}-admin-kubeconfig namespace: {{ $cluster.name }} annotations: argocd.argoproj.io/sync-wave: "0" @@ -28,7 +51,7 @@ spec: - extract: conversionStrategy: Default decodingStrategy: None - key: secret/hub/privatekey + key: secret/hub/{{ $cluster.name }}_cluster_kubeconfig metadataPolicy: None refreshInterval: 24h0m0s secretStoreRef: @@ -37,17 +60,17 @@ spec: target: creationPolicy: Owner deletionPolicy: Retain - name: {{ $cluster.name }}-cluster-private-key + name: admin-kubeconfig template: engineVersion: v2 mergePolicy: Replace type: Opaque - +{{- else }} --- apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: {{ $cluster.name }}-cluster-pull-secret + name: {{ $cluster.name }}-cluster-private-key namespace: {{ $cluster.name }} annotations: argocd.argoproj.io/sync-wave: "0" @@ -56,7 +79,7 @@ spec: - extract: conversionStrategy: Default decodingStrategy: None - key: secret/hub/openshiftPullSecret + key: secret/hub/privatekey metadataPolicy: None refreshInterval: 24h0m0s secretStoreRef: @@ -65,7 +88,7 @@ spec: target: creationPolicy: Owner deletionPolicy: Retain - name: {{ $cluster.name }}-cluster-pull-secret + name: {{ $cluster.name }}-cluster-private-key template: engineVersion: v2 mergePolicy: Replace @@ -75,7 +98,7 @@ spec: apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: {{ $cluster.name }}-cluster-aws-creds + name: {{ $cluster.name }}-cluster-pull-secret namespace: {{ $cluster.name }} annotations: argocd.argoproj.io/sync-wave: "0" @@ -84,7 +107,7 @@ spec: - extract: conversionStrategy: Default decodingStrategy: None - key: secret/hub/aws + key: secret/hub/openshiftPullSecret metadataPolicy: None refreshInterval: 24h0m0s secretStoreRef: @@ -93,7 +116,7 @@ spec: target: creationPolicy: Owner deletionPolicy: Retain - name: {{ $cluster.name }}-cluster-aws-creds + name: {{ $cluster.name }}-cluster-pull-secret template: engineVersion: v2 mergePolicy: Replace @@ -145,8 +168,42 @@ spec: name: img{{ $cluster.version }}-multi-appsub pullSecretRef: name: {{ $cluster.name }}-cluster-pull-secret +{{- end }} {{- /* if .Values.byoc */}} + +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ $cluster.name }}-cluster-aws-creds + namespace: {{ $cluster.name }} + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + dataFrom: + - extract: + conversionStrategy: Default + decodingStrategy: None + key: secret/hub/aws + metadataPolicy: None + refreshInterval: 24h0m0s + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + creationPolicy: Owner + deletionPolicy: Retain + name: {{ $cluster.name }}-cluster-aws-creds + template: + engineVersion: v2 + mergePolicy: Replace + type: Opaque --- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $cluster.name }} +--- apiVersion: cluster.open-cluster-management.io/v1 kind: ManagedCluster metadata: @@ -188,4 +245,4 @@ spec: enabled: true iamPolicyController: enabled: true -{{- end }} +{{- end }} {{- /* range list $effectivePrimary $effectiveSecondary */}} diff --git a/charts/hub/rdr/values.yaml b/charts/hub/rdr/values.yaml index 67f0f5b..f5a2637 100644 --- a/charts/hub/rdr/values.yaml +++ b/charts/hub/rdr/values.yaml @@ -6,6 +6,20 @@ global: edgeGitopsVms: chartVersion: "0.2.10" +# Set this variable to true to use your own clusters as regional clusters +# Also add the kubconfigs to these clusters as secrets to values-secret yaml +# +# - name: ocp-primary_cluster_kubeconfig +# fields: +# - name: kubeconfig +# path: path to kubeconfig for the cluster +# - name: ocp-secondary_cluster_kubeconfig +# fields: +# - name: kubeconfig +# path: path to kubeconfig for the cluster +byoc: false + + # Minimal overrides for cluster names, versions, regions (e.g. overrides/values-cluster-names.yaml). # Merge override file FIRST so full regionalDR stays here and only clusterOverrides are applied. # clusterOverrides: diff --git a/values-hub.yaml b/values-hub.yaml index c06fbf5..1f739a2 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -129,7 +129,10 @@ clusterGroup: # These resources are created by OCM operators and should never be pruned - group: internal.open-cluster-management.io kind: ManagedClusterInfo - + # Uncomment this, and add your cluster kubeconfig to value secrets to use your clusters as DR clusters, instead of creating ones + # overrides: + # - name: byoc + # value: true vault: name: vault namespace: vault diff --git a/values-secret.yaml.template b/values-secret.yaml.template index e4e28fa..f73e32a 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -49,3 +49,14 @@ secrets: fields: - name: .dockerconfigjson value: "A standard OpenShift pull secret" + +# These are needed if you want to use and import your own regional clusters +# by setting byoc: true in rdr chart +# - name: ocp-primary_cluster_kubeconfig +# fields: +# - name: kubeconfig +# path: path to kubeconfig for the cluster +# - name: ocp-secondary_cluster_kubeconfig +# fields: +# - name: kubeconfig +# path: path to kubeconfig for the cluster