Add bgp to support ip address pool on resilient clusters

Makefile

@@ -3,3 +3,20 @@
 # You can add custom targets above or below the include line
 
 include Makefile-common
+
+export WESTCONFIG=$(PWD)/ocp-primary.yaml
+export EASTCONFIG=$(PWD)/ocp-secondary.yaml
+
+##@ AWS Infrastructure tasks
+.PHONY: download-kubeconfigs
+download-kubeconfigs: ## Downloads the kubeconfig for the 2 managedcluster
+	./scripts/download-kubeconfigs.sh
+
+.PHONY: bgp-routing
+bgp-routing:  download-kubeconfigs ## Sets up the BGP routing with a client ec2 in aws
+	cd ansible && ansible-playbook -i hosts $(EXTRA_ARGS) $(EXTRA_VARS) playbooks/router.yml
+
+
+.PHONY: bgp-routing-cleanup
+bgp-routing-cleanup: ## Cleans up the BGP routing with a client ec2 in aws
+	cd ansible && ansible-playbook -i hosts $(EXTRA_ARGS) $(EXTRA_VARS) playbooks/router-cleanup.yml

ansible/ansible.cfg

@@ -2,3 +2,4 @@
 display_skipped_hosts=False
 localhost_warning=False
 roles_path=./roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles
+host_key_checking = False

ansible/group_vars/all

@@ -0,0 +1,141 @@
+---
+aws_profile: "default"
+
+aws_ami_owner: 309956199498
+aws_ami_name: "RHEL-10*"
+aws_ami_arch: "x86_64"
+
+aws_default_tags:
+  pattern: ingress
+
+aws_default_az: ""
+
+ssh_pubkey: "{{ lookup('file', '~/.ssh/id_rsa.pub' | expanduser) }}"
+wait_for_ssh_timeout: 600
+
+hosted_zone: aws.validatedpatterns.io.
+
+acm_import_clustergroup_label: east
+
+metallb_openshift_asn:
+  west: 65001
+  east: 65002
+metallb_address_pool: 192.168.155.0/24
+metallb_bfd_enable: true
+
+ec2_vpcs:
+  coreclient:
+    # 192.168.8.1 - 192.168.11.254
+    cidr: "192.168.8.0/22"
+    subnet_cidr: "192.168.8.0/24"
+    region: west
+
+  corewest:
+    # 192.168.12.1 - 192.168.13.254
+    cidr: "192.168.12.0/23"
+    subnet_cidr: "192.168.12.0/24"
+    region: west
+    tgw_peer: westtor
+
+  westtor:
+    # 192.168.14.1 - 192.168.15.254
+    cidr: "192.168.14.0/23"
+    subnet_cidr: "192.168.14.0/24"
+    region: west
+    tgw_peer: corewest
+
+  coreeast:
+    # 192.168.16.1 - 192.168.17.254
+    cidr: "192.168.16.0/23"
+    subnet_cidr: "192.168.16.0/24"
+    region: west # we need this in the same region as the coreclient, we add a nic here from core vm
+    tgw_peer: easttor
+
+  easttor:
+    # 192.168.18.1 - 192.168.19.254
+    cidr: "192.168.18.0/23"
+    subnet_cidr: "192.168.18.0/24"
+    region: east
+    tgw_peer: coreeast
+
+ec2_vms:
+  client:
+    vpc: coreclient
+    private_ip: 192.168.8.10
+    type: t2.micro
+    extra_nics: []
+
+  core:
+    vpc: coreclient
+    private_ip: 192.168.8.100
+    type: t2.small
+    inventory_groups:
+      - frr
+    extra_nics:
+      - vpc: corewest
+        ip: 192.168.12.200
+        metric: 101
+
+      - vpc: coreeast
+        ip: 192.168.16.200
+        metric: 101
+
+  westtor:
+    vpc: westtor
+    private_ip: 192.168.14.100
+    type: t2.micro
+    extra_nics: []
+    inventory_groups:
+      - frr
+
+  easttor:
+    vpc: easttor
+    private_ip: 192.168.18.100
+    type: t2.micro
+    extra_nics: []
+    inventory_groups:
+      - frr
+
+ec2_frrs:
+  core:
+    asn: 64666
+    connections:
+      towest:
+        remote_ips:
+          - 192.168.14.100
+        local_ip: 192.168.12.200/24
+        asn: 64001
+
+      toeast:
+        remote_ips:
+          - 192.168.18.100
+        local_ip: 192.168.16.200/24
+        asn: 64002
+
+    fixed_advertisements:
+      - 192.168.8.0/24
+
+  westtor:
+    asn: 64001
+    connections:
+      tocore:
+        remote_ips:
+          - 192.168.12.200
+        local_ip: 192.168.14.100/24
+        asn: 64666
+      # We will extend this list dynamically with the ocp workers ips-asn
+    fixed_advertisements: []
+
+  easttor:
+    asn: 64002
+    connections:
+      tocore:
+        remote_ips:
+          - 192.168.16.200
+        local_ip: 192.168.18.100/24
+        asn: 64666
+      # We will extend this list dynamically with the ocp workers ips-asn
+    fixed_advertisements: []
+
+
+allow_different_regions: true

ansible/playbooks/aws_ec2_frr.yml

@@ -0,0 +1,35 @@
+- name: Configure frr on {{ frr.key }}
+  become: true
+  delegate_to: "{{ frr.key }}"
+  block:
+  - name: Create frr config on {{ frr.key }}
+    ansible.builtin.template:
+      src: ../templates/frr.conf.j2
+      dest: /etc/frr/frr.conf
+    notify: Restart frr on frr nodes
+
+  - name: Create frr daemons config on {{ frr.key }}
+    ansible.builtin.template:
+      src: ../templates/daemons.j2
+      dest: /etc/frr/daemons
+    notify: Restart frr on frr nodes
+
+  - name: Create vtysh config on {{ frr.key }}
+    ansible.builtin.template:
+      src: ../templates/vtysh.conf.j2
+      dest: /etc/frr/vtysh.conf
+    notify: Restart frr on frr nodes
+
+  - name: Create vtysh config on {{ frr.key }}
+    ansible.builtin.template:
+      src: ../templates/vtysh.conf.j2
+      dest: /etc/frr/vtysh.conf
+    notify: Restart frr on frr nodes
+
+  - name: "Enable and start the frr Service"
+    ansible.builtin.service:
+      name: frr
+      enabled: true
+      state: started
+    notify: Restart frr on frr nodes
+

ansible/playbooks/check_task.yml

@@ -0,0 +1,107 @@
+# Checks that env variables are set, that clusters are reachable
+# Sets aws_region and {hub,spoke}_cluster_name and {hub,spoke}_fqdn facts
+# Also sets aws_arn_userid and ec2_ssh_key_name
+
+- name: Get AWS caller identity for Owner tag
+  amazon.aws.aws_caller_info:
+    profile: "{{ aws_profile }}"
+  register: aws_caller_info
+
+- name: Set AWS ARN User ID
+  ansible.builtin.set_fact:
+    aws_arn_userid: "{{ aws_caller_info.arn | split('/') | last }}"
+
+- name: Set AWS ec2 ssh key name
+  ansible.builtin.set_fact:
+    ec2_ssh_key_name: "{{ aws_arn_userid }}-ssh-key-rsa"
+
+- name: Set WESTCONFIG and EASTCONFIG facts from env variables
+  ansible.builtin.set_fact:
+    WESTCONFIG: "{{ lookup('env', 'WESTCONFIG') }}"
+    EASTCONFIG: "{{ lookup('env', 'EASTCONFIG') }}"
+
+- name: Check for correct WESTCONFIG variables
+  ansible.builtin.fail:
+    msg: "WESTCONFIG variable needs to be set and pointing to the WEST cluster (with ACM) kubeconfig file"
+  when:
+    WESTCONFIG is not defined or WESTCONFIG | length == 0
+
+- name: Check for correct EASTCONFIG env variable
+  ansible.builtin.fail:
+    msg: "EASTCONFIG variable needs to be set and pointing to the EAST cluster (without ACM) kubeconfig file"
+  when:
+    EASTCONFIG is not defined or EASTCONFIG | length == 0
+
+- name: Show the two cluster kubeconfig paths
+  ansible.builtin.debug:
+    msg: "WESTCONFIG: {{ WESTCONFIG }} - EASTCONFIG: {{ EASTCONFIG }}"
+
+- name: Check that both clusters are reachable
+  ansible.builtin.shell: |
+    oc cluster-info
+  environment:
+    KUBECONFIG: "{{ item }}"
+  loop:
+    - "{{ WESTCONFIG }}"
+    - "{{ EASTCONFIG }}"
+
+- name: Get cluster routes and set facts
+  ansible.builtin.shell: |
+    oc get Ingress.config.openshift.io/cluster -o jsonpath='{.spec.domain}'
+  environment:
+    KUBECONFIG: "{{ item.config }}"
+  register: route_result
+  loop:
+    - { name: "west", config: "{{ WESTCONFIG }}" }
+    - { name: "east", config: "{{ EASTCONFIG }}" }
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Set cluster routes info dynamically
+  ansible.builtin.set_fact:
+    "{{ item.item.name }}_cluster_name": "{{ (item.stdout | split('.'))[1] }}"
+    "{{ item.item.name }}_fqdn": "{{ (item.stdout | split('.'))[1:] | join('.') }}"
+  loop: "{{ route_result.results }}"        
+  loop_control:
+    label: "{{ item.stdout }}"
+
+- name: Get clusters' aws region
+  ansible.builtin.shell: |
+    oc get infrastructure cluster -o jsonpath='{.status.platformStatus.aws.region}'
+  environment:
+    KUBECONFIG: "{{ item.config }}"
+  register: aws_region_result
+  loop:
+    - { name: "west", config: "{{ WESTCONFIG }}" }
+    - { name: "east", config: "{{ EASTCONFIG }}" }
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Set cluster aws region info dynamically
+  ansible.builtin.set_fact:
+    "{{ item.item.name }}_aws_region": "{{ item.stdout }}"
+  loop: "{{ aws_region_result.results }}"        
+  loop_control:
+    label: "{{ item.stdout }}"
+
+- name: Validate clusters are in the same region (unless explicitly overridden)
+  ansible.builtin.assert:
+    that:
+      - west_aws_region == east_aws_region or allow_different_regions | default(false)
+    fail_msg: |
+      FATAL: Clusters are in different regions (west: {{ west_aws_region }}, east: {{ east_aws_region }}).
+      Multi-region support is not fully implemented.
+      If you want to proceed anyway, set 'allow_different_regions: true' in your overrides.yml file.
+      WARNING: This may result in connectivity issues or deployment failures.
+
+- name: Set aws region
+  ansible.builtin.set_fact:
+    aws_region: "{{ west_aws_region }}"
+
+- name: Set AWS AZ
+  ansible.builtin.set_fact:
+    aws_az: "{{ aws_region }}a"
+
+- name: Print AWS infos
+  ansible.builtin.debug:
+    msg: "AWS User: {{ aws_arn_userid }} - AWS Profile: {{ aws_profile }} - AWS Region: {{ aws_region }}"

ansible/playbooks/cleanup_task.yml

@@ -0,0 +1,22 @@
+# This task assumes that it is invoked with the following loop:
+# loop:
+#   - { name: "hub", config: "{{ HUBCONFIG }}", cluster_name: "{{ hub_cluster_name }}" }
+#   - { name: "spoke", config: "{{ SPOKECONFIG }}", cluster_name: "{{ spoke_cluster_name }}" }
+# loop_control:
+#   loop_var: ocp
+
+- name: Delete frr ec2 instance
+  amazon.aws.ec2_instance:
+    profile: "{{ aws_profile }}"
+    region: "{{ aws_region }}"
+    filters:
+      "tag:Name": "frr-ec2-{{ ocp.cluster_name }}"
+      instance-state-name: ["pending","running","shutting-down", "stopping", "stopped"]
+    state: terminated
+
+- name: Delete ec2 security group
+  amazon.aws.ec2_security_group:
+    profile: "{{ aws_profile }}"
+    region: "{{ aws_region }}"
+    name: test-client-{{ ocp.cluster_name }}-security-group
+    state: absent