diff --git a/Makefile b/Makefile index b6b36c3..c800d0f 100644 --- a/Makefile +++ b/Makefile @@ -3,3 +3,20 @@ # You can add custom targets above or below the include line include Makefile-common + +export WESTCONFIG=$(PWD)/ocp-primary.yaml +export EASTCONFIG=$(PWD)/ocp-secondary.yaml + +##@ AWS Infrastructure tasks +.PHONY: download-kubeconfigs +download-kubeconfigs: ## Downloads the kubeconfig for the 2 managedcluster + ./scripts/download-kubeconfigs.sh + +.PHONY: bgp-routing +bgp-routing: download-kubeconfigs ## Sets up the BGP routing with a client ec2 in aws + cd ansible && ansible-playbook -i hosts $(EXTRA_ARGS) $(EXTRA_VARS) playbooks/router.yml + + +.PHONY: bgp-routing-cleanup +bgp-routing-cleanup: ## Cleans up the BGP routing with a client ec2 in aws + cd ansible && ansible-playbook -i hosts $(EXTRA_ARGS) $(EXTRA_VARS) playbooks/router-cleanup.yml \ No newline at end of file diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index c8bec62..b4a333f 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -2,3 +2,4 @@ display_skipped_hosts=False localhost_warning=False roles_path=./roles:~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles +host_key_checking = False diff --git a/ansible/group_vars/all b/ansible/group_vars/all new file mode 100644 index 0000000..6a23737 --- /dev/null +++ b/ansible/group_vars/all @@ -0,0 +1,141 @@ +--- +aws_profile: "default" + +aws_ami_owner: 309956199498 +aws_ami_name: "RHEL-10*" +aws_ami_arch: "x86_64" + +aws_default_tags: + pattern: ingress + +aws_default_az: "" + +ssh_pubkey: "{{ lookup('file', '~/.ssh/id_rsa.pub' | expanduser) }}" +wait_for_ssh_timeout: 600 + +hosted_zone: aws.validatedpatterns.io. + +acm_import_clustergroup_label: east + +metallb_openshift_asn: + west: 65001 + east: 65002 +metallb_address_pool: 192.168.155.0/24 +metallb_bfd_enable: true + +ec2_vpcs: + coreclient: + # 192.168.8.1 - 192.168.11.254 + cidr: "192.168.8.0/22" + subnet_cidr: "192.168.8.0/24" + region: west + + corewest: + # 192.168.12.1 - 192.168.13.254 + cidr: "192.168.12.0/23" + subnet_cidr: "192.168.12.0/24" + region: west + tgw_peer: westtor + + westtor: + # 192.168.14.1 - 192.168.15.254 + cidr: "192.168.14.0/23" + subnet_cidr: "192.168.14.0/24" + region: west + tgw_peer: corewest + + coreeast: + # 192.168.16.1 - 192.168.17.254 + cidr: "192.168.16.0/23" + subnet_cidr: "192.168.16.0/24" + region: west # we need this in the same region as the coreclient, we add a nic here from core vm + tgw_peer: easttor + + easttor: + # 192.168.18.1 - 192.168.19.254 + cidr: "192.168.18.0/23" + subnet_cidr: "192.168.18.0/24" + region: east + tgw_peer: coreeast + +ec2_vms: + client: + vpc: coreclient + private_ip: 192.168.8.10 + type: t2.micro + extra_nics: [] + + core: + vpc: coreclient + private_ip: 192.168.8.100 + type: t2.small + inventory_groups: + - frr + extra_nics: + - vpc: corewest + ip: 192.168.12.200 + metric: 101 + + - vpc: coreeast + ip: 192.168.16.200 + metric: 101 + + westtor: + vpc: westtor + private_ip: 192.168.14.100 + type: t2.micro + extra_nics: [] + inventory_groups: + - frr + + easttor: + vpc: easttor + private_ip: 192.168.18.100 + type: t2.micro + extra_nics: [] + inventory_groups: + - frr + +ec2_frrs: + core: + asn: 64666 + connections: + towest: + remote_ips: + - 192.168.14.100 + local_ip: 192.168.12.200/24 + asn: 64001 + + toeast: + remote_ips: + - 192.168.18.100 + local_ip: 192.168.16.200/24 + asn: 64002 + + fixed_advertisements: + - 192.168.8.0/24 + + westtor: + asn: 64001 + connections: + tocore: + remote_ips: + - 192.168.12.200 + local_ip: 192.168.14.100/24 + asn: 64666 + # We will extend this list dynamically with the ocp workers ips-asn + fixed_advertisements: [] + + easttor: + asn: 64002 + connections: + tocore: + remote_ips: + - 192.168.16.200 + local_ip: 192.168.18.100/24 + asn: 64666 + # We will extend this list dynamically with the ocp workers ips-asn + fixed_advertisements: [] + + +allow_different_regions: true \ No newline at end of file diff --git a/ansible/playbooks/aws_ec2_frr.yml b/ansible/playbooks/aws_ec2_frr.yml new file mode 100644 index 0000000..b025d12 --- /dev/null +++ b/ansible/playbooks/aws_ec2_frr.yml @@ -0,0 +1,35 @@ +- name: Configure frr on {{ frr.key }} + become: true + delegate_to: "{{ frr.key }}" + block: + - name: Create frr config on {{ frr.key }} + ansible.builtin.template: + src: ../templates/frr.conf.j2 + dest: /etc/frr/frr.conf + notify: Restart frr on frr nodes + + - name: Create frr daemons config on {{ frr.key }} + ansible.builtin.template: + src: ../templates/daemons.j2 + dest: /etc/frr/daemons + notify: Restart frr on frr nodes + + - name: Create vtysh config on {{ frr.key }} + ansible.builtin.template: + src: ../templates/vtysh.conf.j2 + dest: /etc/frr/vtysh.conf + notify: Restart frr on frr nodes + + - name: Create vtysh config on {{ frr.key }} + ansible.builtin.template: + src: ../templates/vtysh.conf.j2 + dest: /etc/frr/vtysh.conf + notify: Restart frr on frr nodes + + - name: "Enable and start the frr Service" + ansible.builtin.service: + name: frr + enabled: true + state: started + notify: Restart frr on frr nodes + diff --git a/ansible/playbooks/check_task.yml b/ansible/playbooks/check_task.yml new file mode 100644 index 0000000..8deda47 --- /dev/null +++ b/ansible/playbooks/check_task.yml @@ -0,0 +1,107 @@ +# Checks that env variables are set, that clusters are reachable +# Sets aws_region and {hub,spoke}_cluster_name and {hub,spoke}_fqdn facts +# Also sets aws_arn_userid and ec2_ssh_key_name + +- name: Get AWS caller identity for Owner tag + amazon.aws.aws_caller_info: + profile: "{{ aws_profile }}" + register: aws_caller_info + +- name: Set AWS ARN User ID + ansible.builtin.set_fact: + aws_arn_userid: "{{ aws_caller_info.arn | split('/') | last }}" + +- name: Set AWS ec2 ssh key name + ansible.builtin.set_fact: + ec2_ssh_key_name: "{{ aws_arn_userid }}-ssh-key-rsa" + +- name: Set WESTCONFIG and EASTCONFIG facts from env variables + ansible.builtin.set_fact: + WESTCONFIG: "{{ lookup('env', 'WESTCONFIG') }}" + EASTCONFIG: "{{ lookup('env', 'EASTCONFIG') }}" + +- name: Check for correct WESTCONFIG variables + ansible.builtin.fail: + msg: "WESTCONFIG variable needs to be set and pointing to the WEST cluster (with ACM) kubeconfig file" + when: + WESTCONFIG is not defined or WESTCONFIG | length == 0 + +- name: Check for correct EASTCONFIG env variable + ansible.builtin.fail: + msg: "EASTCONFIG variable needs to be set and pointing to the EAST cluster (without ACM) kubeconfig file" + when: + EASTCONFIG is not defined or EASTCONFIG | length == 0 + +- name: Show the two cluster kubeconfig paths + ansible.builtin.debug: + msg: "WESTCONFIG: {{ WESTCONFIG }} - EASTCONFIG: {{ EASTCONFIG }}" + +- name: Check that both clusters are reachable + ansible.builtin.shell: | + oc cluster-info + environment: + KUBECONFIG: "{{ item }}" + loop: + - "{{ WESTCONFIG }}" + - "{{ EASTCONFIG }}" + +- name: Get cluster routes and set facts + ansible.builtin.shell: | + oc get Ingress.config.openshift.io/cluster -o jsonpath='{.spec.domain}' + environment: + KUBECONFIG: "{{ item.config }}" + register: route_result + loop: + - { name: "west", config: "{{ WESTCONFIG }}" } + - { name: "east", config: "{{ EASTCONFIG }}" } + loop_control: + label: "{{ item.name }}" + +- name: Set cluster routes info dynamically + ansible.builtin.set_fact: + "{{ item.item.name }}_cluster_name": "{{ (item.stdout | split('.'))[1] }}" + "{{ item.item.name }}_fqdn": "{{ (item.stdout | split('.'))[1:] | join('.') }}" + loop: "{{ route_result.results }}" + loop_control: + label: "{{ item.stdout }}" + +- name: Get clusters' aws region + ansible.builtin.shell: | + oc get infrastructure cluster -o jsonpath='{.status.platformStatus.aws.region}' + environment: + KUBECONFIG: "{{ item.config }}" + register: aws_region_result + loop: + - { name: "west", config: "{{ WESTCONFIG }}" } + - { name: "east", config: "{{ EASTCONFIG }}" } + loop_control: + label: "{{ item.name }}" + +- name: Set cluster aws region info dynamically + ansible.builtin.set_fact: + "{{ item.item.name }}_aws_region": "{{ item.stdout }}" + loop: "{{ aws_region_result.results }}" + loop_control: + label: "{{ item.stdout }}" + +- name: Validate clusters are in the same region (unless explicitly overridden) + ansible.builtin.assert: + that: + - west_aws_region == east_aws_region or allow_different_regions | default(false) + fail_msg: | + FATAL: Clusters are in different regions (west: {{ west_aws_region }}, east: {{ east_aws_region }}). + Multi-region support is not fully implemented. + If you want to proceed anyway, set 'allow_different_regions: true' in your overrides.yml file. + WARNING: This may result in connectivity issues or deployment failures. + +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ west_aws_region }}" + +- name: Set AWS AZ + ansible.builtin.set_fact: + aws_az: "{{ aws_region }}a" + +- name: Print AWS infos + ansible.builtin.debug: + msg: "AWS User: {{ aws_arn_userid }} - AWS Profile: {{ aws_profile }} - AWS Region: {{ aws_region }}" diff --git a/ansible/playbooks/cleanup_task.yml b/ansible/playbooks/cleanup_task.yml new file mode 100644 index 0000000..c087322 --- /dev/null +++ b/ansible/playbooks/cleanup_task.yml @@ -0,0 +1,22 @@ +# This task assumes that it is invoked with the following loop: +# loop: +# - { name: "hub", config: "{{ HUBCONFIG }}", cluster_name: "{{ hub_cluster_name }}" } +# - { name: "spoke", config: "{{ SPOKECONFIG }}", cluster_name: "{{ spoke_cluster_name }}" } +# loop_control: +# loop_var: ocp + +- name: Delete frr ec2 instance + amazon.aws.ec2_instance: + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + filters: + "tag:Name": "frr-ec2-{{ ocp.cluster_name }}" + instance-state-name: ["pending","running","shutting-down", "stopping", "stopped"] + state: terminated + +- name: Delete ec2 security group + amazon.aws.ec2_security_group: + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + name: test-client-{{ ocp.cluster_name }}-security-group + state: absent diff --git a/ansible/playbooks/create_ec2.yml b/ansible/playbooks/create_ec2.yml new file mode 100644 index 0000000..eff7064 --- /dev/null +++ b/ansible/playbooks/create_ec2.yml @@ -0,0 +1,73 @@ +- name: Set ami_image and aws_region region + ansible.builtin.set_fact: + aws_region: "{{ vars[ec2_vpcs[instance.value.vpc].region ~ '_aws_region'] }}" + ec2_ami_image_id: "{{ vars[ec2_vpcs[instance.value.vpc].region ~ '_ec2_ami_image_id'] }}" + +- name: Launch EC2 Instance {{ instance.key }}-vm + amazon.aws.ec2_instance: + name: "{{ instance.key }}-vm" + instance_type: "{{ instance.value.type }}" + image_id: "{{ ec2_ami_image_id }}" + key_name: "{{ ec2_ssh_key_name }}" + user_data: "{{ lookup('template', '../templates/cloud_init_userdata.j2') | b64encode }}" + vpc_subnet_id: "{{ ec2_vpcs[instance.value.vpc].subnet_id }}" + security_group: "{{ ec2_vpcs[instance.value.vpc].security_group_id }}" + source_dest_check: false + network: + assign_public_ip: true + private_ip_address: "{{ instance.value.private_ip }}" + state: started + wait: true + tags: "{{ current_tags | combine({ 'Name': instance.key + '-vm' }) }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: ec2_info + +- name: Extend ec2_vms instance facts for {{ instance.key }}-vm + ansible.builtin.set_fact: + ec2_vms: >- + {{ + ec2_vms | combine({ + instance.key: instance.value | combine({ + 'public_ip': ec2_info.instances[0].public_ip_address, + 'private_ip': ec2_info.instances[0].private_ip_address, + 'instance_id': ec2_info.instances[0].instance_id + }) + }, recursive=True) + }} + +- name: Wait for SSH to be available + ansible.builtin.wait_for: + host: "{{ ec2_vms[instance.key].public_ip }}" + port: 22 + timeout: "{{ wait_for_ssh_timeout }}" + state: started + +- name: Add new host to in-memory inventory + ansible.builtin.add_host: + name: "{{ instance.key }}" + ansible_host: "{{ ec2_vms[instance.key].public_ip }}" + groups: "{{ instance.value.vpc | list + instance.value.inventory_groups | default([]) }}" + ansible_user: ec2-user + +- name: Set hostname {{ instance.key }} + become: true + delegate_to: "{{ instance.key }}" + ansible.builtin.hostname: + name: "{{ instance.key }}" + use: systemd + +- name: Add extra nics for {{ instance.key }} + ansible.builtin.include_tasks: create_nic.yml + when: instance.value.extra_nics | length > 0 + loop: "{{ instance.value.extra_nics }}" + loop_control: + loop_var: nic + label: "{{ nic.vpc }} - {{ nic.ip }}" + extended: true + +- name: Enforce rp_filter on all new nics + become: true + delegate_to: "{{ instance.key }}" + ansible.builtin.shell: | + for i in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 0 > $i; done diff --git a/ansible/playbooks/create_nic.yml b/ansible/playbooks/create_nic.yml new file mode 100644 index 0000000..1b786a2 --- /dev/null +++ b/ansible/playbooks/create_nic.yml @@ -0,0 +1,50 @@ +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[ec2_vpcs[nic.vpc].region ~ '_aws_region'] }}" + +- name: Get ENI in spoke's vpc subnet + amazon.aws.ec2_eni_info: + filters: + attachment.instance-id: "{{ ec2_vms[instance.key].instance_id }}" + subnet-id: "{{ ec2_vpcs[nic.vpc].subnet_id }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: ec2_eni_info_raw + +- name: Create ENI in spoke's vpc subnet if not exists + amazon.aws.ec2_eni: + subnet_id: "{{ ec2_vpcs[nic.vpc].subnet_id }}" + security_groups: "{{ ec2_vpcs[nic.vpc].security_group_id }}" + instance_id: "{{ ec2_vms[instance.key].instance_id }}" + device_index: "{{ ansible_loop.index }}" + private_ip_address: "{{ nic.ip }}" + attached: true + tags: "{{ current_tags | combine({ 'Name': instance.key + '-' + nic.vpc + '-nic' }) }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: eni_created + when: ec2_eni_info_raw['network_interfaces'] | length == 0 + +- name: Debug ENI info + ansible.builtin.debug: + msg: "{{ eni_created.interface if (ec2_eni_info_raw['network_interfaces'] | length == 0) else ec2_eni_info_raw['network_interfaces'][0]['id'] }}" + +- name: Make sure we delete the ENI when we remove the ec2 VM + amazon.aws.ec2_eni: + eni_id: "{{ eni_id }}" + source_dest_check: false + delete_on_termination: true + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + vars: + eni_id: "{{ eni_created.interface.id if (ec2_eni_info_raw['network_interfaces'] | length == 0) else ec2_eni_info_raw['network_interfaces'][0]['id'] }}" + +# FIXME(bandini): This assumes that the main nic is not called "Wired Connection" +# experiments with community.general.nmcli failed horribly. The below works +- name: Set metric on the newly created + become: true + delegate_to: "{{ instance.key }}" + ansible.builtin.shell: | + set -e + nmcli c modify "Wired connection {{ ansible_loop.index }}" ipv4.route-metric {{ nic.metric }} + nmcli c up "Wired connection {{ ansible_loop.index }}" diff --git a/ansible/playbooks/create_tgw.yml b/ansible/playbooks/create_tgw.yml new file mode 100644 index 0000000..a57f240 --- /dev/null +++ b/ansible/playbooks/create_tgw.yml @@ -0,0 +1,40 @@ +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[vpc.value.region ~ '_aws_region'] }}" + +- name: "Create a new transit gateway for {{ vpc.key }}" + amazon.aws.ec2_transit_gateway: + state: present + region: "{{ aws_region }}" + description: "{{ vpc.key }} tgw" + tags: + Name: "{{ vpc.key }}" + register: created_tgw + +# Attach tgw to vpc +- name: "Create a Transit Gateway vpc attachment for {{ vpc.key }}" + amazon.aws.ec2_transit_gateway_vpc_attachment: + state: "present" + region: "{{ aws_region }}" + transit_gateway: "{{ created_tgw.transit_gateway.transit_gateway_id }}" + name: "{{ vpc.key }}-tgw-attach" + subnets: + - "{{ vpc.value.subnet_id }}" + ipv6_support: false + tags: + Name: "{{ vpc.key }}-tgw-attach" + register: created_tgw_vpc_attach + +- name: Extend ec2_vpcs variable with newly create info + ansible.builtin.set_fact: + ec2_vpcs: >- + {{ + ec2_vpcs | combine({ + vpc.key: vpc.value | combine({ + 'tgw_id': created_tgw.transit_gateway.transit_gateway_id, + 'tgw_id_owner_id': created_tgw.transit_gateway.owner_id, + 'tgw_route_table_id': created_tgw.transit_gateway.options.association_default_route_table_id, + 'tgw_vpc_attachment_id': created_tgw_vpc_attach.attachments | map(attribute='transit_gateway_attachment_id') | first , + }) + }, recursive=True) + }} diff --git a/ansible/playbooks/create_tgw_routes.yml b/ansible/playbooks/create_tgw_routes.yml new file mode 100644 index 0000000..bf258eb --- /dev/null +++ b/ansible/playbooks/create_tgw_routes.yml @@ -0,0 +1,250 @@ +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[vpc.value.region ~ '_aws_region'] }}" + +- name: Get peering connection (attachment) between tgw-s + ansible.builtin.shell: | + aws ec2 describe-transit-gateway-peering-attachments \ + --region {{ aws_region }} \ + --profile {{ aws_profile }} \ + --filters Name=transit-gateway-id,Values={{ tgw_id }} Name=state,Values=\[available,initiatingRequest,pendingAcceptance,peering\] \ + --output json \ + --no-paginate + register: tgw_peering_conn + vars: + tgw_id: "{{ ec2_vpcs[vpc.key].tgw_id }}" + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + +- name: Create peering connection (attachment) between tgw-s + ansible.builtin.shell: | + aws ec2 create-transit-gateway-peering-attachment \ + --region {{ aws_region }} \ + --profile {{ aws_profile }} \ + --transit-gateway-id {{ ec2_vpcs[vpc.key].tgw_id }} \ + --peer-transit-gateway-id {{ ec2_vpcs[vpc.value.tgw_peer].tgw_id }} \ + --peer-region {{ vars[ec2_vpcs[vpc.value.tgw_peer].region ~ '_aws_region'] }} \ + --peer-account-id {{ ec2_vpcs[vpc.value.tgw_peer].tgw_id_owner_id }} \ + --tag-specifications ResourceType=transit-gateway-attachment,Tags=\[\{Key=Name,Value={{ vpc.key }}-peer-{{ vpc.value.tgw_peer }}-tgw-attach,\}\] \ + --output json \ + --no-paginate + register: tgw_peering_attachments + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + - tgw_peering_conn_list | length == 0 + vars: + tgw_peering_conn_list: "{{ tgw_peering_conn.stdout | from_json | json_query('TransitGatewayPeeringAttachments') }}" + +- name: Get peering connection to accept for {{ vpc.key }}" + ansible.builtin.shell: | + aws ec2 describe-transit-gateway-peering-attachments \ + --region {{ vars[ec2_vpcs[vpc.value.tgw_peer].region ~ '_aws_region'] }} \ + --profile {{ aws_profile }} \ + --filters Name=transit-gateway-id,Values={{ tgw_id }} Name=state,Values=\[pendingAcceptance\] \ + --output json \ + --no-paginate + register: get_tgw_peer_attachment + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + vars: + tgw_id: "{{ ec2_vpcs[vpc.value.tgw_peer].tgw_id }}" + +- name: Accept peering connection between tgw-s {{ vpc.key }} + ansible.builtin.shell: | + aws ec2 accept-transit-gateway-peering-attachment \ + --transit-gateway-attachment-id {{ tgw_attachment_id }} \ + --region {{ vars[ec2_vpcs[vpc.value.tgw_peer].region ~ '_aws_region'] }} + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + - (get_tgw_peer_attachment.stdout | from_json | json_query('TransitGatewayPeeringAttachments')) | length > 0 + vars: + tgw_attachment_id: "{{ get_tgw_peer_attachment.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | first | json_query('TransitGatewayAttachmentId') }}" + +# retry and wait for peering to be available +- name: Get peering attachment id + ansible.builtin.shell: | + aws ec2 describe-transit-gateway-peering-attachments \ + --region {{ aws_region }} \ + --profile {{ aws_profile }} \ + --filters Name=transit-gateway-id,Values={{ tgw_id }} Name=state,Values=\[available\] \ + --output json \ + --no-paginate + register: tgw_peering_conn + until: tgw_peering_conn.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | length > 0 + delay: 20 + retries: 20 + vars: + tgw_id: "{{ vpc.value.tgw_id }}" + +# Recreate the route always +- name: Get route in tgw for peered tgw + ansible.builtin.shell: | + aws ec2 search-transit-gateway-routes \ + --region {{ aws_region }}\ + --filters Name=route-search.exact-match,Values={{ ec2_vpcs[vpc.value.tgw_peer].cidr }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + register: get_peered_route + +- name: Delete route in tgw for peered tgw + ansible.builtin.shell: | + aws ec2 delete-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ ec2_vpcs[vpc.value.tgw_peer].cidr }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + when: get_peered_route.stdout | from_json | json_query('Routes') | length > 0 + +- name: Create route in tgw for peered tgw + ansible.builtin.shell: | + aws ec2 create-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ ec2_vpcs[vpc.value.tgw_peer].cidr }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + --transit-gateway-attachment-id {{ tgw_attachment_id }} + vars: + tgw_attachment_id: "{{ tgw_peering_conn.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | first | json_query('TransitGatewayAttachmentId') }}" + +# Recreate the route always +- name: Get route in tgw for metallb anycast + ansible.builtin.shell: | + aws ec2 search-transit-gateway-routes \ + --region {{ aws_region }}\ + --filters Name=route-search.exact-match,Values={{ metallb_address_pool }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + register: get_metallb_route + +- name: Delete route in tgw for metallb anycast + ansible.builtin.shell: | + aws ec2 delete-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ metallb_address_pool }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + when: get_metallb_route.stdout | from_json | json_query('Routes') | length > 0 + +- name: Create route in tgw for metallb anycast range towards peer attachment if we are in the core* tgw + ansible.builtin.shell: | + aws ec2 create-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ metallb_address_pool }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + --transit-gateway-attachment-id {{ tgw_attachment_id }} + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + vars: + tgw_attachment_id: "{{ tgw_peering_conn.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | first | json_query('TransitGatewayAttachmentId') }}" + +- name: Create route in tgw for metallb anycast range towards vpc attachment if we are in the tor tgw + ansible.builtin.shell: | + aws ec2 create-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ metallb_address_pool }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + --transit-gateway-attachment-id {{ tgw_attachment_id }} + when: + - vpc.key == "westtor" or vpc.key == "easttor" + vars: + tgw_attachment_id: "{{ vpc.value.tgw_vpc_attachment_id }}" + +# Recreate the route always +- name: Get route in tgw for for client cidr + ansible.builtin.shell: | + aws ec2 search-transit-gateway-routes \ + --region {{ aws_region }}\ + --filters Name=route-search.exact-match,Values={{ ec2_vpcs['coreclient']['subnet_cidr'] }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + register: get_client_route + +- name: Delete route in tgw for metallb anycast + ansible.builtin.shell: | + aws ec2 delete-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ ec2_vpcs['coreclient']['subnet_cidr'] }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + when: get_client_route.stdout | from_json | json_query('Routes') | length > 0 + +- name: Create route in tgw for client cidr range towards peer attachment if we are in the tor tgw + ansible.builtin.shell: | + aws ec2 create-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ ec2_vpcs['coreclient']['subnet_cidr'] }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + --transit-gateway-attachment-id {{ tgw_attachment_id }} + when: + - vpc.key == "westtor" or vpc.key == "easttor" + vars: + tgw_attachment_id: "{{ tgw_peering_conn.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | first | json_query('TransitGatewayAttachmentId') }}" + +- name: Create route in tgw for client cidr range towards vpc attachment if we are in the core* tgw + ansible.builtin.shell: | + aws ec2 create-transit-gateway-route \ + --region {{ aws_region }}\ + --destination-cidr-block {{ ec2_vpcs['coreclient']['subnet_cidr'] }} \ + --transit-gateway-route-table-id {{ vpc.value.tgw_route_table_id }} \ + --transit-gateway-attachment-id {{ tgw_attachment_id }} + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + vars: + tgw_attachment_id: "{{ vpc.value.tgw_vpc_attachment_id }}" + +- name: Get nic for core VM + amazon.aws.ec2_eni_info: + filters: + attachment.instance-id: "{{ ec2_vms['core']['instance_id'] }}" + addresses.private-ip-address: "{{ ip_address }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: get_core_vm_nic + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + vars: + ip_address: "{{ (vpc.key == 'corewest') | ternary('192.168.12.200', '192.168.16.200') }}" + +- name: Get nic for tor VM + amazon.aws.ec2_eni_info: + filters: + attachment.instance-id: "{{ ec2_vms[vpc.key]['instance_id'] }}" + addresses.private-ip-address: "{{ ip_address }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: get_tor_vm_nic + when: + - vpc.key == "westtor" or vpc.key == "easttor" + vars: + ip_address: "{{ (vpc.key == 'westtor') | ternary('192.168.14.100', '192.168.18.100') }}" + +- name: Set routes for the tor vpc-s + ansible.builtin.set_fact: + routes: + - dest: "{{ ec2_vpcs[vpc.value.tgw_peer].cidr }}" + transit_gateway_id: "{{ vpc.value.tgw_id }}" + - dest: "{{ metallb_address_pool }}" + network_interface_id: "{{ vm_eni_id }}" + - dest: "{{ ec2_vpcs['coreclient']['subnet_cidr'] }}" + transit_gateway_id: "{{ vpc.value.tgw_id }}" + when: + - vpc.key == "westtor" or vpc.key == "easttor" + vars: + vm_eni_id: "{{ get_tor_vm_nic.network_interfaces | map(attribute='id') | first }}" + +- name: Set routes for the core vpc-s + ansible.builtin.set_fact: + routes: + - dest: "{{ ec2_vpcs[vpc.value.tgw_peer].cidr }}" + transit_gateway_id: "{{ vpc.value.tgw_id }}" + - dest: "{{ metallb_address_pool }}" + transit_gateway_id: "{{ vpc.value.tgw_id }}" + - dest: "{{ ec2_vpcs['coreclient']['subnet_cidr'] }}" + network_interface_id: "{{ vm_eni_id }}" + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + vars: + vm_eni_id: "{{ get_core_vm_nic.network_interfaces | map(attribute='id') | first }}" + +- name: Create route in vpc rtbl for metallb, peer, and client + amazon.aws.ec2_vpc_route_table: + vpc_id: "{{ vpc.value.vpc_id }}" + route_table_id: "{{ vpc.value.route_table_id }}" + lookup: id + purge_routes: false + routes: "{{ routes }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" diff --git a/ansible/playbooks/create_vpc.yml b/ansible/playbooks/create_vpc.yml new file mode 100644 index 0000000..fbd3e66 --- /dev/null +++ b/ansible/playbooks/create_vpc.yml @@ -0,0 +1,84 @@ +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[instance.value.region ~ '_aws_region'] }}" + +- name: Set AWS AZ + ansible.builtin.set_fact: + aws_az: "{{ aws_region }}a" + +- name: Create VPC {{ instance.key }}-vpc + amazon.aws.ec2_vpc_net: + name: "{{ instance.key }}-vpc" + cidr_block: "{{ instance.value.cidr }}" + tags: "{{ current_tags | combine({ 'Name': instance.key + '-vpc'}) }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: vpc_result + +- name: Create Internet Gateway {{ instance.key }}-igw + amazon.aws.ec2_vpc_igw: + vpc_id: "{{ vpc_result.vpc.id }}" + tags: "{{ current_tags | combine({ 'Name': instance.key + '-igw' }) }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: igw_result + +- name: Create Subnet {{ instance.key }}-net + amazon.aws.ec2_vpc_subnet: + az: "{{ aws_az }}" + vpc_id: "{{ vpc_result.vpc.id }}" + cidr: "{{ instance.value.subnet_cidr }}" + map_public: true + tags: "{{ current_tags | combine({ 'Name': instance.key + '-net' }) }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: subnet_result + +- name: Create and Associate Route Table {{ instance.key }}-rtbl + amazon.aws.ec2_vpc_route_table: + vpc_id: "{{ vpc_result.vpc.id }}" + subnets: + - "{{ subnet_result.subnet.id }}" + routes: + - dest: 0.0.0.0/0 + gateway_id: "{{ igw_result.gateway_id }}" + tags: "{{ current_tags | combine({ 'Name': instance.key + '-rtbl' }) }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: rtbl_result + +- name: Create Security Group {{ instance.key }}-sg + amazon.aws.ec2_security_group: + name: "{{ instance.key }}-sg" + description: "Allow SSH" + vpc_id: "{{ vpc_result.vpc.id }}" + rules: + - proto: tcp + ports: 22 + cidr_ip: 0.0.0.0/0 + - proto: all + cidr_ip: 10.0.0.0/8 + rule_desc: All port from vpc + - proto: all + cidr_ip: 192.168.0.0/16 + rule_desc: Allow client-core-tors + tags: "{{ current_tags | combine({ 'Name': instance.key + '-sg' }) }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: sg_result + +# FIXME(bandini): Is there really no better way to extend an existing (or even a new) +# dictionary with new info?? +- name: Extend ec2_vpcs variable with newly create info + ansible.builtin.set_fact: + ec2_vpcs: >- + {{ + ec2_vpcs | combine({ + instance.key: instance.value | combine({ + 'vpc_id': vpc_result.vpc.id, + 'route_table_id': rtbl_result.route_table.id, + 'subnet_id': subnet_result.subnet.id, + 'security_group_id': sg_result.group_id + }) + }, recursive=True) + }} diff --git a/ansible/playbooks/delete_ec2.yml b/ansible/playbooks/delete_ec2.yml new file mode 100644 index 0000000..ce8e229 --- /dev/null +++ b/ansible/playbooks/delete_ec2.yml @@ -0,0 +1,28 @@ +- name: Set ami_image and aws_region region + ansible.builtin.set_fact: + aws_region: "{{ vars[ec2_vpcs[instance.value.vpc].region ~ '_aws_region'] }}" + +- name: Gather the ec2 instance details for {{ instance.key }} + amazon.aws.ec2_instance_info: + filters: + "tag:Name": "{{ instance.key }}-vm" + "tag:washere": "{{ aws_arn_userid }}" + instance-state-name: ["pending", "running", "shutting-down", "stopping", "stopped"] + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: ec2_node_info + +- name: Delete ec2 instance if it exists {{ instance.key }} + when: ec2_node_info.instances | length != 0 + block: + - name: Set ec2 instance id for {{ instance.key }} + ansible.builtin.set_fact: + instance_id: "{{ ec2_node_info.instances | map(attribute='instance_id') | first }}" + + - name: Delete ec2 instance {{ instance.key }} + amazon.aws.ec2_instance: + instance_ids: + - "{{ instance_id }}" + state: terminated + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" diff --git a/ansible/playbooks/delete_tgw.yml b/ansible/playbooks/delete_tgw.yml new file mode 100644 index 0000000..80f50c1 --- /dev/null +++ b/ansible/playbooks/delete_tgw.yml @@ -0,0 +1,73 @@ +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[vpc.value.region ~ '_aws_region'] }}" + +- name: "Get a Transit Gateway vpc attachment for {{ vpc.key }}" + amazon.aws.ec2_transit_gateway_vpc_attachment_info: + region: "{{ aws_region }}" + name: "{{ vpc.key }}-tgw-attach" + register: get_tgw_vpc_attachment + +- name: "Delete a Transit Gateway vpc attachment for {{ vpc.key }}" + amazon.aws.ec2_transit_gateway_vpc_attachment: + state: absent + region: "{{ aws_region }}" + name: "{{ vpc.key }}-tgw-attach" + when: + - get_tgw_vpc_attachment.attachments | length > 0 + +- name: "Get a Transit Gateway peer attachment for {{ vpc.key }}" + ansible.builtin.shell: | + aws ec2 describe-transit-gateway-peering-attachments \ + --region {{ aws_region }} \ + --profile {{ aws_profile }} \ + --filters Name=tag:Name,Values=\[{{ vpc.key }}-peer-{{ vpc.value.tgw_peer }}-tgw-attach\] \ + --output json \ + --no-paginate + register: get_tgw_peer_attachment + vars: + tgw_id: "{{ ec2_vpcs[vpc.key].tgw_id }}" + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + +- name: "Delete a Transit Gateway peer attachment for {{ vpc.key }}" + ansible.builtin.shell: | + aws ec2 delete-transit-gateway-peering-attachment \ + --region {{ aws_region }} \ + --profile {{ aws_profile }} \ + --transit-gateway-attachment-id {{ tgw_peer_attachment_ids | first }} \ + --output json + vars: + tgw_id: "{{ ec2_vpcs[vpc.key].tgw_id }}" + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + - tgw_peer_attachment_ids | length > 0 + vars: + tgw_peer_attachment_ids: "{{ get_tgw_peer_attachment.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | map(attribute='TransitGatewayAttachmentId') }}" + +- name: "Wait Transit Gateway peer attachment for {{ vpc.key }} to be deleted" + ansible.builtin.shell: | + aws ec2 describe-transit-gateway-peering-attachments \ + --region {{ aws_region }} \ + --profile {{ aws_profile }} \ + --filters Name=tag:Name,Values=\[{{ vpc.key }}-peer-{{ vpc.value.tgw_peer }}-tgw-attach\] Name=state,Values=\[available,initiatingRequest,pendingAcceptance,peering,deleting\] \ + --output json \ + --no-paginate + register: get_tgw_peer_attachment + vars: + tgw_peer_attachment_ids: "{{ get_tgw_peer_attachment.stdout | from_json | json_query('TransitGatewayPeeringAttachments') | map(attribute='TransitGatewayAttachmentId') }}" + tgw_id: "{{ ec2_vpcs[vpc.key].tgw_id }}" + when: + - vpc.key == "corewest" or vpc.key == "coreeast" + until: tgw_peer_attachment_ids | length == 0 + delay: 20 + retries: 15 + +- name: "Delete a new transit gateway for {{ vpc.key }}" + amazon.aws.ec2_transit_gateway: + state: absent + region: "{{ aws_region }}" + description: "{{ vpc.key }} tgw" + tags: + Name: "{{ vpc.key }}" + register: created_tgw diff --git a/ansible/playbooks/delete_vpc.yml b/ansible/playbooks/delete_vpc.yml new file mode 100644 index 0000000..9a6652a --- /dev/null +++ b/ansible/playbooks/delete_vpc.yml @@ -0,0 +1,120 @@ +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[instance.value.region ~ '_aws_region'] }}" + +- name: Find VPC ID by Tag {{ instance.key }} + amazon.aws.ec2_vpc_net_info: + filters: + "tag:Name": "{{ instance.key }}-vpc" + "tag:washere": "{{ aws_arn_userid }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: vpc_info + +- name: Debug + ansible.builtin.debug: + msg: "{{ vpc_info }}" + +- name: Find elastic IP info + amazon.aws.ec2_eip_info: + filters: + "tag:Name": "{{ instance.key }}-eip" + "tag:washere": "{{ aws_arn_userid }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: eip_info + +- name: Release Elastic IP + amazon.aws.ec2_eip: + state: absent + allocation_id: "{{ eip_info.addresses[0].allocation_id }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + when: eip_info.addresses | length > 0 + +- name: Fetch existing Security Groups called {{ instance.key }}-sg + amazon.aws.ec2_security_group_info: + filters: + "group_name": "{{ instance.key }}-sg" + "tag:washere": "{{ aws_arn_userid }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: sg_info + +- name: Delete Security Groups + amazon.aws.ec2_security_group: + state: absent + group_id: "{{ item.group_id }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + loop: "{{ sg_info.security_groups }}" + loop_control: + label: "{{ item.group_id }}" + when: sg_info.security_groups | length > 0 + +- name: Find existing subnets called {{ instance.key }}-net + amazon.aws.ec2_vpc_subnet_info: + filters: + "tag:Name": "{{ instance.key }}-net" + "tag:washere": "{{ aws_arn_userid }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: subnet_info + +- name: Debug 2 + ansible.builtin.debug: + msg: "{{ subnet_info }}" + +- name: Delete Subnets + amazon.aws.ec2_vpc_subnet: + state: absent + cidr: "{{ item.cidr_block }}" + vpc_id: "{{ item.vpc_id }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + loop: "{{ subnet_info.subnets }}" + loop_control: + label: "{{ item.cidr_block }}" + when: subnet_info.subnets | length > 0 + +- name: Find Route Tables called {{ instance.key }}-rtbl + amazon.aws.ec2_vpc_route_table_info: + filters: + "tag:Name": "{{ instance.key }}-rtbl" + "tag:washere": "{{ aws_arn_userid }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + register: rtbl_info + +- name: Debug 3 + ansible.builtin.debug: + msg: "{{ rtbl_info }}" + +- name: Delete Route Tables + amazon.aws.ec2_vpc_route_table: + state: absent + route_table_id: "{{ item.route_table_id }}" + lookup: id + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + loop: "{{ rtbl_info.route_tables }}" + loop_control: + label: "{{ item.route_table_id }}" + when: rtbl_info.route_tables | length > 0 + +- name: Delete Internet Gateway for vpc {{ instance.key }} + amazon.aws.ec2_vpc_igw: + state: absent + vpc_id: "{{ vpc_info.vpcs[0].vpc_id }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + when: vpc_info.vpcs | length > 0 + +- name: Delete VPC {{ instance.key }} + amazon.aws.ec2_vpc_net: + state: absent + vpc_id: "{{ vpc_info.vpcs[0].vpc_id }}" + cidr_block: "{{ instance.value.cidr }}" + region: "{{ aws_region }}" + profile: "{{ aws_profile }}" + when: vpc_info.vpcs | length > 0 diff --git a/ansible/playbooks/get_ami.yml b/ansible/playbooks/get_ami.yml new file mode 100644 index 0000000..c5b0808 --- /dev/null +++ b/ansible/playbooks/get_ami.yml @@ -0,0 +1,24 @@ +# Fetches the AMI image id, prints it and returns it as the "ec2_ami_image_id" +# fact. This AMI is used across all EC2 instances +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[item ~ '_aws_region'] }}" + +- name: Get ami image + ansible.builtin.shell: | + aws --profile {{ aws_profile }} ec2 describe-images \ + --owners {{ aws_ami_owner }} \ + --filters "Name=name,Values={{ aws_ami_name }}*" "Name=state,Values=available" "Name=architecture,Values={{ aws_ami_arch }}" \ + --region {{ aws_region }} \ + --query 'sort_by(Images, &CreationDate)[-1].ImageId' \ + --output text + register: ec2_ami_image_id_raw + changed_when: false + +- name: Set ami image fact + ansible.builtin.set_fact: + "{{ item }}_ec2_ami_image_id": "{{ ec2_ami_image_id_raw.stdout }}" + +- name: Print ami image use + ansible.builtin.debug: + msg: "Using AMI image: {{ vars[item ~ '_ec2_ami_image_id'] }} for {{ item }} region" diff --git a/ansible/playbooks/import-cluster.yml b/ansible/playbooks/import-cluster.yml new file mode 100644 index 0000000..a0ecdfd --- /dev/null +++ b/ansible/playbooks/import-cluster.yml @@ -0,0 +1,28 @@ +--- +- name: Importing ACM clusters automatically + hosts: localhost + connection: local + pre_tasks: + - name: Set WESTCONFIG and EASTCONFIG facts from env variables + ansible.builtin.set_fact: + WESTCONFIG: "{{ lookup('env', 'WESTCONFIG') }}" + EASTCONFIG: "{{ lookup('env', 'EASTCONFIG') }}" + + - name: Check for correct WESTCONFIG variable + ansible.builtin.fail: + msg: "WESTCONFIG variable needs to be set and pointing to the HUB kubeconfig file" + when: + WESTCONFIG is not defined or WESTCONFIG | length == 0 + + - name: Check for correct EASTCONFIG env variable + ansible.builtin.fail: + msg: "EASTCONFIG variable needs to be set and pointing to the REGION kubeconfig file" + when: + EASTCONFIG is not defined or EASTCONFIG | length == 0 + + - name: Print the hub and regional kubeconfigs + ansible.builtin.debug: + msg: "WESTCONFIG: {{ WESTCONFIG }} - EASTCONFIG: {{ EASTCONFIG }}" + + roles: + - acm_import diff --git a/ansible/playbooks/network_task.yml b/ansible/playbooks/network_task.yml new file mode 100644 index 0000000..8ce3ae0 --- /dev/null +++ b/ansible/playbooks/network_task.yml @@ -0,0 +1,341 @@ +# This task assumes that it is invoked with the following loop: +# loop: +# - { +# name: "west", +# config: "{{ WESTCONFIG }}", +# cluster_name: "{{ west_cluster_name }}" +# tor_vm: "westtor" +# } +# - { +# name: "east", +# config: "{{ EASTCONFIG }}", +# cluster_name: "{{ east_cluster_name }}" +# tor_vm: "easttor" +# } +# loop_control: +# loop_var: ocp +- name: Set aws region + ansible.builtin.set_fact: + aws_region: "{{ vars[ocp.name ~ '_aws_region'] }}" + +- name: Set AWS AZ + ansible.builtin.set_fact: + aws_az: "{{ aws_region }}a" + + +- name: Fetch the config-v1 ConfigMap from kube-system + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + name: cluster-config-v1 + namespace: kube-system + kubeconfig: "{{ ocp.config }}" + register: cluster_config_v1 + +- name: Extract MachineNetwork CIDR + ansible.builtin.set_fact: + target_cidr: >- + {{ + (cluster_config_v1.resources[0].data['install-config'] | from_yaml).networking.machineNetwork[0].cidr + }} + when: cluster_config_v1.resources | length > 0 + +- name: Show CIDR for "{{ ocp.name }}" + ansible.builtin.debug: + msg: "The Machine Network CIDR is: {{ target_cidr }}" + +- name: Get worker node ip list for "{{ ocp.name }}" + ansible.builtin.shell: | + set -e -o pipefail + oc describe nodes -l node-role.kubernetes.io/worker | grep InternalIP: | awk '{ print $2 }' + environment: + KUBECONFIG: "{{ ocp.config }}" + register: worker_nodes + +- name: Get master node ip list for "{{ ocp.name }}" + ansible.builtin.shell: | + set -e -o pipefail + oc describe nodes -l node-role.kubernetes.io/master | grep InternalIP: | awk '{ print $2 }' + environment: + KUBECONFIG: "{{ ocp.config }}" + register: master_nodes + +- name: Gather security group info for workers for "{{ ocp.name }}" + amazon.aws.ec2_security_group_info: + filters: + "tag:sigs.k8s.io/cluster-api-provider-aws/role": "node" + "tag:Name": "{{ ocp.cluster_name }}-*" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: ec2_security_group_info + +- name: Debug for "{{ ocp.name }} - {{ ocp.cluster_name }}" + ansible.builtin.debug: + msg: "ec2_security_group_info: {{ ec2_security_group_info }}" + +- name: Set vpc and sg id for "{{ ocp.name }}" + ansible.builtin.set_fact: + sg_vpc_id: "{{ ec2_security_group_info.security_groups[0].vpc_id }}" + sg_group_id: "{{ ec2_security_group_info.security_groups[0].group_id }}" + sg_group_name: "{{ ec2_security_group_info.security_groups[0].group_name }}" + sg_worker_description: "{{ ec2_security_group_info.security_groups[0].description }}" + +- name: Gather vpc subnet info for workers for "{{ ocp.name }}" + amazon.aws.ec2_vpc_subnet_info: + filters: + vpc-id: "{{ sg_vpc_id }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: ec2_vpc_subnet_info + +# only add the private subnet, and there will be only exactly one +- name: Set private subnet id for "{{ ocp.name }}" + ansible.builtin.set_fact: + private_subnet_id: "{{ ec2_vpc_subnet_info.subnets | selectattr('map_public_ip_on_launch', '==', false) | selectattr('availability_zone', '==', aws_az) | map(attribute='subnet_id') | first }}" + private_subnet_cidr: "{{ ec2_vpc_subnet_info.subnets | selectattr('map_public_ip_on_launch', '==', false) | selectattr('availability_zone', '==', aws_az) | map(attribute='cidr_block') | first }}" + +- name: Print out the the private subnet for "{{ ocp.name }} + ansible.builtin.debug: + msg: + private_subnet_id: "{{ private_subnet_id }}" + +- name: Open up worker node security groups + amazon.aws.ec2_security_group: + name: "{{ sg_group_name }}" + description: "{{ sg_worker_description }}" + rules: + - proto: all + cidr_ip: "{{ target_cidr }}" + rule_desc: All port from vpc + - proto: all + cidr_ip: 192.168.0.0/16 + rule_desc: Allow client-core-tors + purge_rules: false + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + +- name: Get NICs for a vpc + amazon.aws.ec2_eni_info: + filters: + vpc-id: "{{ sg_vpc_id }}" + interface-type: interface + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: ec2_eni_info + +# Modify the each worker node interface to disable source/dest check (needed +# for the ip address not on the vpc) +- name: Disable source/dest check on worker node NIC-s + amazon.aws.ec2_eni: + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + source_dest_check: false + eni_id: "{{ item['id'] }}" + when: item['private_ip_address'] in worker_nodes.stdout_lines + loop: "{{ ec2_eni_info['network_interfaces'] }}" + +- name: Create ec2 security group + amazon.aws.ec2_security_group: + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + name: "{{ ocp.tor_vm }}-{{ ocp.cluster_name }}-security-group" + description: sec group for client + tags: "{{ current_tags | combine({ 'Name': ocp.tor_vm + '-' + ocp.name + '-sg' }) }}" + vpc_id: "{{ sg_vpc_id }}" + rules: + - proto: all + cidr_ip: 192.168.0.0/16 + rule_desc: All traffic from client core and tors + - proto: all + cidr_ip: "{{ target_cidr }}" + rule_desc: All port from vpc + - proto: tcp + from_port: 22 + to_port: 22 + cidr_ip: 0.0.0.0/0 + rule_desc: allow ssh access from the internet + +- name: Get ENI in TOR {{ ocp.name }} subnet for {{ ocp.tor_vm }} + amazon.aws.ec2_eni_info: + filters: + attachment.instance-id: "{{ ec2_vms[ocp.tor_vm].instance_id }}" + subnet-id: "{{ private_subnet_id }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: ec2_eni_info_raw + +- name: Calculate last usable address in private_subnet + ansible.builtin.set_fact: + last_usable_address: "{{ private_subnet_cidr | ansible.utils.ipaddr('last_usable') }}" + +- name: Print last public address for subnet {{ private_subnet_id }} + ansible.builtin.debug: + msg: "{{ last_usable_address }}" + +- name: Assert that the ip won't clash with any worker ip + ansible.builtin.assert: + that: + - "last_usable_address not in worker_nodes.stdout_lines" + - "last_usable_address not in master_nodes.stdout_lines" + fail_msg: "{{ last_usable_address }} collided with {{ worker_nodes.stdout_lines }}" + +- name: Create ENI in TOR {{ ocp.tor_vm }} if it does not exists + amazon.aws.ec2_eni: + subnet_id: "{{ private_subnet_id }}" + security_groups: "{{ sg_group_id }}" + instance_id: "{{ ec2_vms[ocp.tor_vm].instance_id }}" + private_ip_address: "{{ last_usable_address }}" + device_index: 1 + attached: true + tags: "{{ current_tags | combine({ 'Name': ocp.tor_vm + '-' + ocp.name + '-nic' }) }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: eni_created + when: ec2_eni_info_raw['network_interfaces'] | length == 0 + +- name: Debug additional ENI interface id + ansible.builtin.debug: + msg: + ec2_eni_info_raw: "{{ ec2_eni_info_raw }}" + eni_created: "{{ eni_created }}" + eni_id: "{{ eni_created.interface.id if (ec2_eni_info_raw['network_interfaces'] | length == 0) else ec2_eni_info_raw['network_interfaces'][0]['id'] }}" + +- name: Make sure we delete the ENI when we remove the ec2 VM + amazon.aws.ec2_eni: + eni_id: "{{ eni_id }}" + source_dest_check: false + delete_on_termination: true + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + vars: + eni_id: "{{ eni_created.interface.id if (ec2_eni_info_raw['network_interfaces'] | length == 0) else ec2_eni_info_raw['network_interfaces'][0]['id'] }}" + retries: 4 + delay: 10 + +- name: Wait for enX1 + become: true + delegate_to: "{{ ocp.tor_vm }}" + ansible.builtin.shell: | + set -e -o pipefail + python3 -c "import ipaddress, sys; iface = ipaddress.ip_interface(sys.argv[1]); \ + print(iface.network.network_address + 1)" $(ip -o -4 addr show enX1 | awk '{print $4}') + retries: 5 + delay: 3 + +# FIXME(bandini): ideally we check the connection name before +# We calculate the gateway as Network Address + 1. If google is to be +# believed that is always true on AWS. Some with the enX1 nic name +- name: Drop second default route and add route to reach "{{ ocp.name }}" workers + become: true + delegate_to: "{{ ocp.tor_vm }}" + ansible.builtin.shell: | + set -e -o pipefail + GW=$(python3 -c "import ipaddress, sys; iface = ipaddress.ip_interface(sys.argv[1]); \ + print(iface.network.network_address + 1)" $(ip -o -4 addr show enX1 | awk '{print $4}')) + echo "Found GW: ${GW}" + nmcli c modify "Wired connection 1" ipv4.never-default yes + nmcli c modify "Wired connection 1" +ipv4.routes "{{ target_cidr }} ${GW}" + nmcli c up "Wired connection 1" + +- name: Gather Facts manually on {{ ocp.tor_vm }} + become: true + delegate_to: "{{ ocp.tor_vm }}" + delegate_facts: true + ansible.builtin.setup: + +- name: Set secondary NIC (towards workers) ip address fact + ansible.builtin.set_fact: + enx1_ip: "{{ hostvars[ocp.tor_vm]['ansible_enX1']['ipv4']['address']}}" + "{{ ocp.name }}_enx1_ip": "{{ hostvars[ocp.tor_vm]['ansible_enX1']['ipv4']['address']}}" + +- name: Create dynamic connection dictionary + ansible.builtin.set_fact: + new_connection: >- + {{ + { + ocp.tor_vm: { + 'connections': { + 'to' ~ ocp.name: { + 'asn': metallb_openshift_asn[ocp.name], + 'local_ip': enx1_ip, + 'remote_ips': worker_nodes.stdout_lines + } + } + } + } + }} + +- name: Add new connection info to ec2_frrs + ansible.builtin.set_fact: + ec2_frrs: "{{ ec2_frrs | combine(new_connection, recursive=True) }}" + +- name: Print updated ec2_frrs + ansible.builtin.debug: + msg: "{{ ec2_frrs }}" + +# Next three tasks tweak the route table towards the client network and the +# core-west-east network in the ocp workers private subnet +- name: Get route table for client ec2 private subnet + amazon.aws.ec2_vpc_route_table_info: + filters: + association.subnet-id: "{{ private_subnet_id }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: get_priv_route_tables_raw + +- name: Set up route table for client ec2 to ocp woker node instance + amazon.aws.ec2_vpc_route_table: + vpc_id: "{{ sg_vpc_id }}" + route_table_id: "{{ rtb_id }}" + lookup: id + purge_routes: false + routes: + - dest: "{{ ec2_vpcs['coreclient']['subnet_cidr'] }}" + network_interface_id: "{{ tor_to_ocp_eni_id }}" + - dest: "{{ ec2_vpcs[ocp.vpc]['subnet_cidr'] }}" + network_interface_id: "{{ tor_to_ocp_eni_id }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + vars: + rtb_id: "{{ get_priv_route_tables_raw['route_tables'] | map(attribute='route_table_id') | first }}" + tor_to_ocp_eni_id: "{{ eni_created.interface.id if (ec2_eni_info_raw['network_interfaces'] | length == 0) else ec2_eni_info_raw['network_interfaces'][0]['id'] }}" + +# Next two tasks tweak the route table of the west/east vpc +# - name: Get route table for vpc {{ ocp.vpc }} +# amazon.aws.ec2_vpc_route_table_info: +# filters: +# association.subnet-id: "{{ ec2_vpcs[ocp.vpc]['subnet_id'] }}" +# profile: "{{ aws_profile }}" +# region: "{{ aws_region }}" +# register: get_vpc_route_table_raw + +- name: Get ENI in TOR {{ ocp.name }} tor-core subnet + amazon.aws.ec2_eni_info: + filters: + attachment.instance-id: "{{ ec2_vms[ocp.tor_vm]['instance_id'] }}" + subnet-id: "{{ ec2_vpcs[ocp.vpc]['subnet_id'] }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + register: tor_main_eni_raw + +- debug: + msg: "{{ tor_main_eni_raw }}" + +- debug: + msg: "{{ ec2_vpcs[ocp.vpc] }}" + +- name: Set up route table for vpc {{ ocp.vpc }} + amazon.aws.ec2_vpc_route_table: + vpc_id: "{{ ec2_vpcs[ocp.vpc]['vpc_id'] }}" + route_table_id: "{{ rtb_id }}" + lookup: id + purge_routes: false + routes: + - dest: "{{ target_cidr }}" + network_interface_id: "{{ tor_eni_id }}" + profile: "{{ aws_profile }}" + region: "{{ aws_region }}" + vars: + rtb_id: "{{ ec2_vpcs[ocp.vpc]['route_table_id'] }}" + tor_eni_id: "{{ tor_main_eni_raw['network_interfaces'][0]['id'] }}" + when: tor_main_eni_raw['network_interfaces'] | length > 0 diff --git a/ansible/playbooks/router-cleanup.yml b/ansible/playbooks/router-cleanup.yml new file mode 100644 index 0000000..b3146c5 --- /dev/null +++ b/ansible/playbooks/router-cleanup.yml @@ -0,0 +1,44 @@ +--- +- name: Playbook to cleanup up the BGP routing with a client ec2 in aws + hosts: localhost + gather_facts: false + become: false + vars_files: + # Use this to override stuff that won't be committed to git + - ../overrides.yml + + tasks: + - name: Checks + ansible.builtin.include_tasks: check_task.yml + + - name: Delete created key pair + amazon.aws.ec2_key: + profile: "{{ aws_profile }}" + region: "{{ item }}" + name: "{{ ec2_ssh_key_name }}" + state: absent + loop: + - "{{ east_aws_region }}" + - "{{ west_aws_region }}" + + - name: Delete all EC2s + ansible.builtin.include_tasks: delete_ec2.yml + loop: "{{ ec2_vms | dict2items }}" + loop_control: + loop_var: instance + label: "{{ instance.key }}" + + - name: Delete tgws + ansible.builtin.include_tasks: delete_tgw.yml + loop: "{{ ec2_vpcs | dict2items }}" + loop_control: + loop_var: vpc + label: "{{ vpc.key }}" + when: vpc.key != "coreclient" + + - name: Delete all VPCs + ansible.builtin.include_tasks: delete_vpc.yml + loop: "{{ ec2_vpcs | dict2items }}" + loop_control: + loop_var: instance + label: "{{ instance.key }}" diff --git a/ansible/playbooks/router.yml b/ansible/playbooks/router.yml new file mode 100644 index 0000000..e5e4610 --- /dev/null +++ b/ansible/playbooks/router.yml @@ -0,0 +1,152 @@ +--- +- name: Playbook to set up the BGP routing with a client ec2 in aws + hosts: localhost + gather_facts: false + become: false + vars_files: + # Use this to override stuff that won't be committed to git + - ../overrides.yml + + tasks: + - name: Checks for ocp clusters and sets aws_region, aws_az and aws_arn_userid + ansible.builtin.include_tasks: check_task.yml + + - name: Fetch AMI id and set ec2_ami_image_id + ansible.builtin.include_tasks: get_ami.yml + loop: + - west + - east + + - name: Create key pair to access ec2 with ssh + amazon.aws.ec2_key: + profile: "{{ aws_profile }}" + region: "{{ item }}" + name: "{{ ec2_ssh_key_name }}" + key_material: "{{ ssh_pubkey }}" + loop: + - "{{ east_aws_region }}" + - "{{ west_aws_region }}" + + - name: Set tags to use in every object + ansible.builtin.set_fact: + current_tags: "{{ aws_default_tags | combine({ 'washere': aws_arn_userid }) }}" + + - name: Create all the VPCs, subnets, igw and rtbls + ansible.builtin.include_tasks: create_vpc.yml + loop: "{{ ec2_vpcs | dict2items }}" + loop_control: + loop_var: instance + label: "{{ instance.key }}" + + - name: Print VPCs + ansible.builtin.debug: + msg: "{{ ec2_vpcs }}" + + - name: Create transit gateways with vpc attachments + ansible.builtin.include_tasks: create_tgw.yml + loop: "{{ ec2_vpcs | dict2items }}" + loop_control: + loop_var: vpc + label: "{{ vpc.key }}" + when: vpc.key != "coreclient" + + - name: Create all the EC2 instances + ansible.builtin.include_tasks: create_ec2.yml + loop: "{{ ec2_vms | dict2items }}" + loop_control: + loop_var: instance + label: "{{ instance.key }}" + + - name: Configure networking for west and east clusters + ansible.builtin.include_tasks: network_task.yml + loop: + - { + name: "west", + config: "{{ WESTCONFIG }}", + cluster_name: "{{ west_cluster_name }}", + tor_vm: "westtor", + vpc: "corewest" + } + - { + name: "east", + config: "{{ EASTCONFIG }}", + cluster_name: "{{ east_cluster_name }}", + tor_vm: "easttor", + vpc: "coreeast" + } + loop_control: + loop_var: ocp + label: "{{ ocp.name }}" + + - name: Create transit gateways peering, routes + ansible.builtin.include_tasks: create_tgw_routes.yml + loop: "{{ ec2_vpcs | dict2items }}" + loop_control: + loop_var: vpc + label: "{{ vpc.key }}" + when: vpc.key != "coreclient" + + - name: Drop default routes from core + become: true + delegate_to: core + ansible.builtin.shell: | + set -e -o pipefail + nmcli c modify "Wired connection 1" ipv4.never-default yes + nmcli c up "Wired connection 1" + nmcli c modify "Wired connection 2" ipv4.never-default yes + nmcli c up "Wired connection 2" + + - name: Get ENI in core in the coreclient vpc/subnet + amazon.aws.ec2_eni_info: + filters: + addresses.private-ip-address: "{{ ec2_vms['core']['private_ip'] }}" + profile: "{{ aws_profile }}" + region: "{{ vars[ec2_vpcs['coreclient']['region'] ~ '_aws_region'] }}" + register: core_main_eni_raw + + - name: On the coreclient routing table add a route for the metallb network to the core's main nic + amazon.aws.ec2_vpc_route_table: + vpc_id: "{{ ec2_vpcs['coreclient']['vpc_id'] }}" + route_table_id: "{{ rtb_id }}" + lookup: id + purge_routes: false + routes: + - dest: "{{ metallb_address_pool }}" + network_interface_id: "{{ core_client_eni_id }}" + profile: "{{ aws_profile }}" + region: "{{ vars[ec2_vpcs['coreclient']['region'] ~ '_aws_region'] }}" + vars: + rtb_id: "{{ ec2_vpcs['coreclient']['route_table_id'] }}" + core_client_eni_id: "{{ core_main_eni_raw['network_interfaces'][0]['id'] }}" + + - name: Configure FRR on core, torwest and torweast + ansible.builtin.include_tasks: aws_ec2_frr.yml + loop: "{{ ec2_frrs | dict2items }}" + loop_control: + loop_var: frr + label: "{{ frr.key }}" + + - name: Generate tmux shell template + ansible.builtin.template: + src: ../templates/launch_tmux.sh.j2 + dest: "/tmp/launch_tmux.sh" + mode: '0755' + + - name: Print EC2 VMs + ansible.builtin.debug: + msg: "{{ ec2_vms }}" + + # - name: Print TORs enX1 IPs + # ansible.builtin.debug: + # msg: + # WEST ip: "{{ west_enx1_ip }}" + # EAST ip: "{{ east_enx1_ip }}" + + handlers: + - name: Restart frr on frr nodes + ansible.builtin.service: + name: frr + state: restarted + delegate_to: "{{ item }}" + become: true + loop: "{{ groups['frr'] }}" diff --git a/ansible/templates/cloud_init_userdata.j2 b/ansible/templates/cloud_init_userdata.j2 new file mode 100644 index 0000000..24bf837 --- /dev/null +++ b/ansible/templates/cloud_init_userdata.j2 @@ -0,0 +1,22 @@ +#cloud-config +chpasswd: + expire: false +user: ec2-user +packages: +- tmux +- nc +- traceroute +- tcpdump +- vim +- frr +write_files: + - path: /etc/sysctl.d/90-frr-routing.conf + content: | + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + net.ipv4.conf.all.rp_filter = 0 + net.ipv4.conf.default.rp_filter = 0 + +runcmd: + - sysctl --system + - for i in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 0 > $i; done diff --git a/ansible/templates/daemons.j2 b/ansible/templates/daemons.j2 new file mode 100644 index 0000000..6864f3e --- /dev/null +++ b/ansible/templates/daemons.j2 @@ -0,0 +1,86 @@ +# This file tells the frr package which daemons to start. +# +# Sample configurations for these daemons can be found in +# /usr/share/doc/frr/examples/. +# +# ATTENTION: +# +# When activating a daemon for the first time, a config file, even if it is +# empty, has to be present *and* be owned by the user and group "frr", else +# the daemon will not be started by /etc/init.d/frr. The permissions should +# be u=rw,g=r,o=. +# When using "vtysh" such a config file is also needed. It should be owned by +# group "frrvty" and set to ug=rw,o= though. Check /etc/pam.d/frr, too. +# +# The watchfrr, zebra and staticd daemons are always started. +# +bgpd=yes # Enable bgp +ospfd=no +ospf6d=no +ripd=no +ripngd=no +isisd=no +pimd=no +ldpd=no +nhrpd=no +eigrpd=no +babeld=no +sharpd=no +pbrd=no +{% if metallb_bfd_enable %} +bfdd=yes # Enable bfd +{% else %} +bfdd=no +{% endif -%} +fabricd=no +vrrpd=no +pathd=no +# +# If this option is set the /etc/init.d/frr script automatically loads +# the config via "vtysh -b" when the servers are started. +# Check /etc/pam.d/frr if you intend to use "vtysh"! +# +# +vtysh_enable=yes +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" +ospfd_options=" -A 127.0.0.1" +ospf6d_options=" -A ::1" +ripd_options=" -A 127.0.0.1" +ripngd_options=" -A ::1" +isisd_options=" -A 127.0.0.1" +pimd_options=" -A 127.0.0.1" +ldpd_options=" -A 127.0.0.1" +nhrpd_options=" -A 127.0.0.1" +eigrpd_options=" -A 127.0.0.1" +babeld_options=" -A 127.0.0.1" +sharpd_options=" -A 127.0.0.1" +pbrd_options=" -A 127.0.0.1" +staticd_options="-A 127.0.0.1" +bfdd_options=" -A 127.0.0.1" +fabricd_options="-A 127.0.0.1" +vrrpd_options=" -A 127.0.0.1" +pathd_options=" -A 127.0.0.1" +# configuration profile +# +#frr_profile="traditional" +#frr_profile="datacenter" +# +# This is the maximum number of FD's that will be available. +# Upon startup this is read by the control files and ulimit +# is called. Uncomment and use a reasonable value for your +# setup if you are expecting a large number of peers in +# say BGP. +MAX_FDS=1024 +# The list of daemons to watch is automatically generated by the init script. +#watchfrr_options="" +# To make watchfrr create/join the specified netns, use the following option: +#watchfrr_options="--netns" +# This only has an effect in /etc/frr//daemons, and you need to +# start FRR with "/usr/lib/frr/frrinit.sh start ". +# for debugging purposes, you can specify a "wrap" command to start instead +# of starting the daemon directly, e.g. to use valgrind on ospfd: +# ospfd_wrap="/usr/bin/valgrind" +# or you can use "all_wrap" for all daemons, e.g. to use perf record: +# all_wrap="/usr/bin/perf record --call-graph -" +# the normal daemon command is added to this at the end. \ No newline at end of file diff --git a/ansible/templates/frr.conf.j2 b/ansible/templates/frr.conf.j2 new file mode 100644 index 0000000..29ee4b0 --- /dev/null +++ b/ansible/templates/frr.conf.j2 @@ -0,0 +1,75 @@ +frr defaults traditional +log file /var/log/frr/frr.log +log timestamp precision 3 +! +debug zebra nht +debug bgp neighbor-events +debug bgp nht +debug bgp updates in +debug bgp updates out +{% if metallb_bfd_enable %} +debug bfd peer +{% endif -%} + +ip prefix-list ACCEPT_LAB seq 5 permit {{ metallb_address_pool }} le 32 +{% set ns = namespace(count=10) %} +{% for key, value in ec2_vpcs.items() %} +ip prefix-list ACCEPT_LAB seq {{ ns.count }} permit {{ value.subnet_cidr }} le 32 +{% set ns.count = ns.count + 5 %} +{% endfor %} + +{% for key,connection in frr.value.connections.items() %} +{% for ip in connection.remote_ips %} +ip route {{ ip }}/32 {{ connection.local_ip | ansible.utils.ipaddr('network/prefix') | ansible.utils.ipaddr('next_usable') }} +{% endfor %} +{% endfor %} + +! +router bgp {{ frr.value.asn }} + timers bgp 3 15 + no bgp ebgp-requires-policy + no bgp default ipv4-unicast + no bgp network import-check + bgp bestpath as-path multipath-relax + +{% for key,connection in frr.value.connections.items() %} + neighbor {{ key }} peer-group + neighbor {{ key }} remote-as {{ connection.asn }} +{% for ip in connection.remote_ips %} + neighbor {{ ip }} peer-group {{ key }} + neighbor {{ ip }} bfd + neighbor {{ ip }} ebgp-multihop 10 + neighbor {{ ip }} disable-connected-check +{% endfor %} +{% endfor %} + + address-family ipv4 unicast +{% for adv_routes in frr.value.fixed_advertisements %} + network {{ adv_routes }} +{% endfor %} +{% for key,connection in frr.value.connections.items() %} +{% for ip in connection.remote_ips %} + neighbor {{ ip }} next-hop-self + neighbor {{ ip }} soft-reconfiguration inbound + neighbor {{ ip }} activate +# neighbor {{ ip }} route-map ACCEPT_LAB in +{% endfor %} +{% endfor %} + maximum-paths 4 +exit-address-family +! +route-map ACCEPT_LAB permit 10 + match ip address prefix-list ACCEPT_LAB +exit +! +{% if metallb_bfd_enable %} +bfd +{% for key, connection in frr.value.connections.items() %} +{% for ip in connection.remote_ips %} + peer {{ ip }} multihop local-address {{ connection.local_ip }} +{% endfor %} +{% endfor %} +exit +{% endif %} +! +exit diff --git a/ansible/templates/launch_tmux.sh.j2 b/ansible/templates/launch_tmux.sh.j2 new file mode 100644 index 0000000..0845f63 --- /dev/null +++ b/ansible/templates/launch_tmux.sh.j2 @@ -0,0 +1,24 @@ +#!/bin/bash + +SESSION="ansible_lab" + +# Check if session exists +tmux has-session -t $SESSION 2>/dev/null + +if [ $? != 0 ]; then + # Create new session (detached) + tmux new-session -d -s $SESSION + + {% for name, info in ec2_vms.items() %} + # Create window for {{ name }} + tmux new-window -t $SESSION -n "{{ name }}" + tmux send-keys -t $SESSION:"{{ name }}" "ssh ec2-user@{{ info.public_ip }}" C-m + {% endfor %} + + # Kill the default window created (usually index 0 or 1 depending on config) + # or simply select the first one we created. + tmux select-window -t $SESSION:"{{ ec2_vms.keys() | list | first }}" +fi + +# Attach to session +tmux attach -t $SESSION diff --git a/ansible/templates/vtysh.conf.j2 b/ansible/templates/vtysh.conf.j2 new file mode 100644 index 0000000..e69de29 diff --git a/charts/hub/opp/scripts/argocd-health-monitor-cron.sh b/charts/hub/opp/scripts/argocd-health-monitor-cron.sh index a069570..76d12b0 100755 --- a/charts/hub/opp/scripts/argocd-health-monitor-cron.sh +++ b/charts/hub/opp/scripts/argocd-health-monitor-cron.sh @@ -41,12 +41,12 @@ check_cluster_wedged() { local cluster_argocd_instance="" case "$cluster" in "$PRIMARY_CLUSTER") - cluster_argocd_namespace="ramendr-starter-kit-resilient" - cluster_argocd_instance="resilient-gitops-server" + cluster_argocd_namespace="ramendr-starter-kit-res-primary" + cluster_argocd_instance="res-primary-gitops-server" ;; "$SECONDARY_CLUSTER") - cluster_argocd_namespace="ramendr-starter-kit-resilient" - cluster_argocd_instance="resilient-gitops-server" + cluster_argocd_namespace="ramendr-starter-kit-res-secondary" + cluster_argocd_instance="res-secondary-gitops-server" ;; "local-cluster") cluster_argocd_namespace="openshift-gitops" diff --git a/charts/hub/opp/scripts/argocd-health-monitor.sh b/charts/hub/opp/scripts/argocd-health-monitor.sh index 4f3ea48..d42fcd5 100755 --- a/charts/hub/opp/scripts/argocd-health-monitor.sh +++ b/charts/hub/opp/scripts/argocd-health-monitor.sh @@ -41,12 +41,12 @@ check_cluster_wedged() { local cluster_argocd_instance="" case "$cluster" in "$PRIMARY_CLUSTER") - cluster_argocd_namespace="ramendr-starter-kit-resilient" - cluster_argocd_instance="resilient-gitops-server" + cluster_argocd_namespace="ramendr-starter-kit-res-primary" + cluster_argocd_instance="res-primary-gitops-server" ;; "$SECONDARY_CLUSTER") - cluster_argocd_namespace="ramendr-starter-kit-resilient" - cluster_argocd_instance="resilient-gitops-server" + cluster_argocd_namespace="ramendr-starter-kit-res-secondary" + cluster_argocd_instance="res-secondary-gitops-server" ;; "local-cluster") cluster_argocd_namespace="openshift-gitops" diff --git a/charts/hub/rdr/values.yaml b/charts/hub/rdr/values.yaml index 67f0f5b..00e812d 100644 --- a/charts/hub/rdr/values.yaml +++ b/charts/hub/rdr/values.yaml @@ -55,8 +55,8 @@ regionalDR: clusters: # Pair of clusters, make sure to create each in a different region primary: name: ocp-primary - clusterGroup: resilient - version: 4.18.7 + clusterGroup: res-primary + version: 4.19.1 install_config: apiVersion: v1 baseDomain: "{{ join \".\" (slice (splitList \".\" $.Values.global.clusterDomain) 1) }}" @@ -74,6 +74,8 @@ regionalDR: platform: aws: type: m5.metal + zones: + - eu-central-1a networking: clusterNetwork: - cidr: 10.132.0.0/14 @@ -85,7 +87,7 @@ regionalDR: - 172.20.0.0/16 platform: aws: - region: us-west-1 + region: eu-central-1 userTags: project: ValidatedPatterns publish: External @@ -93,8 +95,8 @@ regionalDR: pullSecret: "" secondary: name: ocp-secondary - clusterGroup: resilient - version: 4.18.7 + clusterGroup: res-secondary + version: 4.19.1 install_config: apiVersion: v1 baseDomain: "{{ join \".\" (slice (splitList \".\" $.Values.global.clusterDomain) 1) }}" @@ -112,6 +114,8 @@ regionalDR: platform: aws: type: m5.metal + zones: + - eu-west-3a networking: clusterNetwork: - cidr: 10.136.0.0/14 @@ -123,7 +127,7 @@ regionalDR: - 172.21.0.0/16 platform: aws: - region: us-east-1 + region: eu-west-3 userTags: project: ValidatedPatterns publish: External diff --git a/charts/resilient/metallb/Chart.yaml b/charts/resilient/metallb/Chart.yaml new file mode 100644 index 0000000..8fde2fa --- /dev/null +++ b/charts/resilient/metallb/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +description: A Helm chart to configure metallb +keywords: +- pattern +name: metallb +version: 0.0.1 diff --git a/charts/resilient/metallb/templates/addresspool.yaml b/charts/resilient/metallb/templates/addresspool.yaml new file mode 100644 index 0000000..0e94db5 --- /dev/null +++ b/charts/resilient/metallb/templates/addresspool.yaml @@ -0,0 +1,13 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + namespace: metallb-system + name: sample-addresspool +spec: + addresses: + - 192.168.155.150/32 + - 192.168.155.151/32 + - 192.168.155.152/32 + - 192.168.155.153/32 + - 192.168.155.154/32 + - 192.168.155.155/32 diff --git a/charts/resilient/metallb/templates/bfdprofile.yaml b/charts/resilient/metallb/templates/bfdprofile.yaml new file mode 100644 index 0000000..6440e6c --- /dev/null +++ b/charts/resilient/metallb/templates/bfdprofile.yaml @@ -0,0 +1,10 @@ +{{- if .Values.metallb.bfd.enabled }} +apiVersion: metallb.io/v1beta1 +kind: BFDProfile +metadata: + name: testbfdprofile + namespace: metallb-system +spec: + receiveInterval: {{ .Values.metallb.bfd.receiveInterval }} + transmitInterval: {{ .Values.metallb.bfd.receiveInterval }} +{{- end }} diff --git a/charts/resilient/metallb/templates/bgpadvertisement.yaml b/charts/resilient/metallb/templates/bgpadvertisement.yaml new file mode 100644 index 0000000..48ac802 --- /dev/null +++ b/charts/resilient/metallb/templates/bgpadvertisement.yaml @@ -0,0 +1,5 @@ +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: example + namespace: metallb-system \ No newline at end of file diff --git a/charts/resilient/metallb/templates/bgppeer.yaml b/charts/resilient/metallb/templates/bgppeer.yaml new file mode 100644 index 0000000..4bbecc0 --- /dev/null +++ b/charts/resilient/metallb/templates/bgppeer.yaml @@ -0,0 +1,16 @@ +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: sample + namespace: metallb-system +spec: + myASN: {{ .Values.metallb.myASN }} + peerASN: {{ .Values.metallb.peerASN }} + peerAddress: {{ .Values.metallb.peerAddress }} +{{- if .Values.metallb.bfd.enabled }} + bfdProfile: testbfdprofile +{{- end }} + nodeSelectors: + - matchLabels: + node-role.kubernetes.io/worker: "" + ebgpMultiHop: true diff --git a/charts/resilient/metallb/templates/metallb.yaml b/charts/resilient/metallb/templates/metallb.yaml new file mode 100644 index 0000000..eaceb88 --- /dev/null +++ b/charts/resilient/metallb/templates/metallb.yaml @@ -0,0 +1,9 @@ +apiVersion: metallb.io/v1beta1 +kind: MetalLB +metadata: + name: metallb + namespace: metallb-system +spec: + logLevel: debug + nodeSelector: + node-role.kubernetes.io/worker: "" diff --git a/charts/resilient/metallb/values.yaml b/charts/resilient/metallb/values.yaml new file mode 100644 index 0000000..5516d82 --- /dev/null +++ b/charts/resilient/metallb/values.yaml @@ -0,0 +1,13 @@ +--- +# global: +# hubClusterDomain: hub.example.com +# localCluster: local.example.com + +metallb: + myASN: 65001 + peerASN: 64001 + peerAddress: 10.0.0.0 + bfd: + enabled: true + receiveInterval: 380 + transmitInterval: 270 diff --git a/overrides/values-cluster-names.yaml b/overrides/values-cluster-names.yaml index 31e1a1f..a23db8d 100644 --- a/overrides/values-cluster-names.yaml +++ b/overrides/values-cluster-names.yaml @@ -6,23 +6,23 @@ clusterOverrides: primary: - name: ocp-primary - version: 4.18.7 + name: ocp-primary-aeros + version: 4.19.1 install_config: metadata: - name: ocp-primary + name: ocp-primary-aeros platform: aws: - region: us-west-1 + region: eu-central-1 secondary: - name: ocp-secondary - version: 4.18.7 + name: ocp-secondary-aeros + version: 4.19.1 install_config: metadata: - name: ocp-secondary + name: ocp-secondary-aeros platform: aws: - region: us-east-1 + region: eu-west-3 # DRPC overrides (preferredCluster defaults to primary if unset) drpc: diff --git a/values-hub.yaml b/values-hub.yaml index c06fbf5..4a75476 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -158,11 +158,19 @@ clusterGroup: jobs: [] managedClusterGroups: - resilient: - name: resilient + res-primary: + name: res-primary helmOverrides: - name: clusterGroup.isHubCluster value: "false" acmlabels: - name: clusterGroup - value: resilient + value: res-primary + res-secondary: + name: res-secondary + helmOverrides: + - name: clusterGroup.isHubCluster + value: "false" + acmlabels: + - name: clusterGroup + value: res-secondary \ No newline at end of file diff --git a/values-resilient.yaml b/values-res-primary.yaml similarity index 85% rename from values-resilient.yaml rename to values-res-primary.yaml index 194f30f..fdf1fab 100644 --- a/values-resilient.yaml +++ b/values-res-primary.yaml @@ -1,6 +1,6 @@ --- clusterGroup: - name: resilient + name: res-primary isHubCluster: false argoCD: @@ -20,6 +20,9 @@ clusterGroup: - openshift-workload-availability: operatorGroup: true targetNamespaces: + - metallb-system: + operatorGroup: true + targetNamespaces: [] subscriptions: openshift-virtualization: @@ -49,7 +52,12 @@ clusterGroup: openshift-oadp: name: redhat-oadp-operator namespace: openshift-adp - channel: stable-1.4 + channel: stable + + metallb: + name: metallb-operator + namespace: metallb-system + channel: stable projects: - resilient @@ -131,4 +139,17 @@ clusterGroup: project: resilient path: charts/resilient/data-protection-application + metallb: + name: metallb + namespace: metallb-system + project: resilient + path: charts/resilient/metallb + overrides: + - name: metallb.myASN + value: 65001 + - name: metallb.peerASN + value: 64001 + - name: metallb.peerAddress + value: 10.1.31.254 + managedClusterGroups: [] diff --git a/values-res-secondary.yaml b/values-res-secondary.yaml new file mode 100644 index 0000000..0be507c --- /dev/null +++ b/values-res-secondary.yaml @@ -0,0 +1,155 @@ +--- +clusterGroup: + name: res-secondary + isHubCluster: false + + argoCD: + resourceTrackingMethod: annotation + + namespaces: + - golang-external-secrets + - openshift-cnv + - openshift-storage + - policies + - gitops-vms + - external-dns-operator + - openshift-adp: + operatorGroup: true + targetNamespaces: + - openshift-adp + - openshift-workload-availability: + operatorGroup: true + targetNamespaces: + - metallb-system: + operatorGroup: true + targetNamespaces: [] + + subscriptions: + openshift-virtualization: + name: kubevirt-hyperconverged + namespace: openshift-cnv + channel: stable + + external-dns: + name: external-dns-operator + namespace: external-dns-operator + channel: stable-v1 + + node-health-check: + name: node-healthcheck-operator + namespace: openshift-workload-availability + channel: stable + + self-node-remediation: + name: self-node-remediation + namespace: openshift-workload-availability + channel: stable + + openshift-data-foundation: + name: odf-operator + namespace: openshift-storage + + openshift-oadp: + name: redhat-oadp-operator + namespace: openshift-adp + channel: stable + + metallb: + name: metallb-operator + namespace: metallb-system + channel: stable + + projects: + - resilient + + imperative: + jobs: + - name: clean-golden-images + playbook: ansible/odf_fix_dataimportcrons.yml + verbosity: -vvv + clusterRoleYaml: + - apiGroups: + - "*" + resources: + - persistentvolumeclaims + - datavolumes + - dataimportcrons + - datasources + verbs: + - "*" + - apiGroups: + - "*" + resources: + - "*" + verbs: + - get + - list + - watch + + applications: + ensure-openshift-console-plugins: + name: ensure-openshift-console-plugins + namespace: openshift-console + project: resilient + chart: ensure-openshift-console-plugins + chartVersion: 0.1.* + extraValueFiles: + - '$patternref/overrides/values-console-plugins-spokes.yaml' + + golang-external-secrets: + name: golang-external-secrets + namespace: golang-external-secrets + project: resilient + chart: golang-external-secrets + chartVersion: 0.1.* + + openshift-cnv: + name: openshift-cnv + namespace: openshift-cnv + project: resilient + chart: openshift-virtualization-instance + chartVersion: 0.1.* + + odf: + name: odf + namespace: openshift-storage + project: resilient + chart: openshift-data-foundations + chartVersion: 0.2.* + extraValueFiles: + - '$patternref/overrides/values-odf-chart.yaml' + + node-health-check-operator: + name: node-health-check-operator + namespace: openshift-workload-availability + project: resilient + chart: node-health-check-operator + chartVersion: 0.1.* + + external-dns-operator: + name: external-dns-operator + namespace: external-dns-operator + project: resilient + chart: external-dns-operator + chartVersion: 0.1.* + + adp-data-protection: + name: adp-data-protection-application + namespace: openshift-adp + project: resilient + path: charts/resilient/data-protection-application + + metallb: + name: metallb + namespace: metallb-system + project: resilient + path: charts/resilient/metallb + overrides: + - name: metallb.myASN + value: 65002 + - name: metallb.peerASN + value: 64002 + - name: metallb.peerAddress + value: 10.2.31.254 + + managedClusterGroups: []