vault: Remove dependency on HAProxy in HCP Vault playbooks

markgoddard · markgoddard · commit f593df758ae0 · 2024-06-07T09:28:56.000+01:00
This change modifies the overcloud HashiCorp Vault playbooks to use the
local Vault service rather than via HAProxy. This makes it possible to
deploy and use Vault without HAProxy. This eliminates the previous
bootstrapping issue where HAProxy needed to be deployed without TLS
enabled while generating initial certificates.

To make this work in environments with a proxy configured, https_proxy
is overridden.
diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst
@@ -84,47 +84,6 @@ Setup Vault on the seed node
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud.key
 
-Setup HAProxy config for Vault
-------------------------------
-
-1. Create the HAProxy config to reverse proxy the Vault HA container
-
-   Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes.
-
-   Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg
-
-   .. code-block::
-
-      # Delete "verify none" if not using self-signed/unknown issuer
-      {% raw %}
-      frontend vault_front
-         mode tcp
-         option tcplog
-         bind {{ kolla_internal_vip_address }}:8200
-         default_backend vault_back
-
-      backend vault_back
-         mode tcp
-         option httpchk GET /v1/sys/health
-         # https://www.vaultproject.io/api-docs/system/health
-         # 200: initialized, unsealed, and active
-         # 501: not initialised (required for bootstrapping)
-         # 503: sealed (required for bootstrapping)
-         http-check expect rstatus (200|501|503)
-
-      {% for host in groups['control'] %}
-      {% set host_name = hostvars[host].ansible_facts.hostname %}
-      {% set host_ip = 'api' | kolla_address(host) %}
-         server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5
-      {% endfor %}
-      {% endraw %}
-
-2. Deploy HAProxy with the new Vault service configuration:
-
-   .. code-block::
-
-      kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
-
 Setup Vault HA on the overcloud hosts
 -------------------------------------
 
@@ -215,6 +174,56 @@ Create the backend TLS and RabbitMQ TLS certificates
 
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/<controller>-key.pem
 
+.. _vault-haproxy:
+
+HAProxy integration
+===================
+
+It is possible to expose the overcloud Vault service via the Kolla Ansible HAProxy load balancer.
+This provides a single highly available API endpoint, as well as monitoring of the Vault backends when combined with Prometheus.
+HAProxy integration is no longer required for generating OpenStack control plane certificates, making it possible to deploy Vault and generate certificates before any containers have been deployed by Kolla Ansible.
+
+1. Create the HAProxy config to reverse proxy the Vault HA container
+
+   Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes.
+
+   Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg
+
+   .. code-block::
+
+      # Delete "verify none" if not using self-signed/unknown issuer
+      {% raw %}
+      frontend vault_front
+         mode tcp
+         option tcplog
+         bind {{ kolla_internal_vip_address }}:8200
+         default_backend vault_back
+
+      backend vault_back
+         mode tcp
+         option httpchk GET /v1/sys/health
+         # https://www.vaultproject.io/api-docs/system/health
+         # 200: initialized, unsealed, and active
+         # 501: not initialised (required for bootstrapping)
+         # 503: sealed (required for bootstrapping)
+         http-check expect rstatus (200|501|503)
+
+      {% for host in groups['control'] %}
+      {% set host_name = hostvars[host].ansible_facts.hostname %}
+      {% set host_ip = 'api' | kolla_address(host) %}
+         server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5
+      {% endfor %}
+      {% endraw %}
+
+2. If HAProxy has not yet been deployed, continue to :ref:`certificates deployment <vault-certificates>`.
+   If HAProxy has been deployed, it may be redeployed with the new Vault service configuration:
+
+   .. code-block::
+
+      kayobe overcloud service deploy -kt haproxy
+
+.. _vault-certificates:
+
 Certificates deployment
 =======================
 
@@ -289,6 +298,8 @@ Enable the required TLS variables in kayobe and kolla
 Barbican integration
 ====================
 
+Barbican integration depends on :ref:`HAProxy integration <vault-haproxy>`.
+
 Enable Barbican in kayobe
 -------------------------
 
diff --git a/etc/kayobe/ansible/vault-deploy-barbican.yml b/etc/kayobe/ansible/vault-deploy-barbican.yml
@@ -4,7 +4,7 @@
   gather_facts: True
   hosts: controllers[0]
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
   tasks:
     - name: Assert that secrets_barbican_approle_secret_id is defined
@@ -25,79 +25,82 @@
         extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
         virtualenv: "{{ virtualenv_path }}/kayobe"
 
-    - name: Enable AppRole auth module
-      hashivault_auth_method:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        method_type: approle
-        state: enabled
+    - environment:
+        https_proxy: ''
+      block:
+        - name: Enable AppRole auth module
+          hashivault_auth_method:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            method_type: approle
+            state: enabled
 
-    - name: Enable barbican kv store
-      hashivault_secret_engine:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        name: barbican
-        backend: kv
-        description: "Barbican kv store"
+        - name: Enable barbican kv store
+          hashivault_secret_engine:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            name: barbican
+            backend: kv
+            description: "Barbican kv store"
 
-    - name: Ensure barbican policy is defined
-      hashivault_policy:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        name: "barbican-policy"
-        state: present
-        rules: |
-          path "barbican/*" {
-            capabilities = ["create", "read", "update", "delete", "list"]
-          }
+        - name: Ensure barbican policy is defined
+          hashivault_policy:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            name: "barbican-policy"
+            state: present
+            rules: |
+              path "barbican/*" {
+                capabilities = ["create", "read", "update", "delete", "list"]
+              }
 
-    - name: Ensure barbican AppRole is defined
-      hashivault_approle_role:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        bind_secret_id: true
-        secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}"
-        secret_id_ttl: 0
-        token_policies: barbican-policy
-        name: barbican
+        - name: Ensure barbican AppRole is defined
+          hashivault_approle_role:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            bind_secret_id: true
+            secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}"
+            secret_id_ttl: 0
+            token_policies: barbican-policy
+            name: barbican
 
-    - name: Get barbican Approle ID
-      hashivault_approle_role_id:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        name: barbican
-      register: barbican_role_id
+        - name: Get barbican Approle ID
+          hashivault_approle_role_id:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            name: barbican
+          register: barbican_role_id
 
-    - name: Print barbican Approle ID
-      debug:
-        msg: "barbican role id is {{ barbican_role_id.id }}"
+        - name: Print barbican Approle ID
+          debug:
+            msg: "barbican role id is {{ barbican_role_id.id }}"
 
-    - name: Write barbican Approle ID to file if requested
-      delegate_to: localhost
-      copy:
-        content: "{{ barbican_role_id.id }}"
-        dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}"
-      when: stackhpc_write_barbican_role_id_to_file | default(false) | bool
+        - name: Write barbican Approle ID to file if requested
+          delegate_to: localhost
+          copy:
+            content: "{{ barbican_role_id.id }}"
+            dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}"
+          when: stackhpc_write_barbican_role_id_to_file | default(false) | bool
 
-    - name: Check if barbican Approle Secret ID is defined
-      hashivault_approle_role_secret_get:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        secret: "{{ secrets_barbican_approle_secret_id }}"
-        name: barbican
-      register: barbican_approle_secret_get
+        - name: Check if barbican Approle Secret ID is defined
+          hashivault_approle_role_secret_get:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            secret: "{{ secrets_barbican_approle_secret_id }}"
+            name: barbican
+          register: barbican_approle_secret_get
 
-    - name: Ensure barbican AppRole Secret ID is defined
-      hashivault_approle_role_secret:
-        url: "{{ vault_api_addr }}"
-        ca_cert: "{{ vault_ca_cert }}"
-        token: "{{ vault_keys.root_token }}"
-        secret: "{{ secrets_barbican_approle_secret_id }}"
-        name: barbican
-      when: barbican_approle_secret_get.status == "absent"
+        - name: Ensure barbican AppRole Secret ID is defined
+          hashivault_approle_role_secret:
+            url: "{{ vault_api_addr }}"
+            ca_cert: "{{ vault_ca_cert }}"
+            token: "{{ vault_keys.root_token }}"
+            secret: "{{ secrets_barbican_approle_secret_id }}"
+            name: barbican
+          when: barbican_approle_secret_get.status == "absent"
diff --git a/etc/kayobe/ansible/vault-deploy-overcloud.yml b/etc/kayobe/ansible/vault-deploy-overcloud.yml
@@ -85,9 +85,12 @@
     - import_role:
         name: stackhpc.hashicorp.vault_unseal
       vars:
-        # NOTE: Need to unseal each backend, so don't use the VIP
-        vault_api_addr: "http://localhost:8200"
+        vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
+        vault_unseal_token: "{{ vault_keys.root_token }}"
+        vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
         vault_unseal_keys: "{{ vault_keys.keys_base64 }}"
+      environment:
+        https_proxy: ''
 
 - name: Configure PKI
   any_errors_fatal: true
@@ -108,3 +111,5 @@
         vault_pki_intermediate_roles: "{{ overcloud_vault_pki_roles }}"
         vault_pki_write_certificate_files: true
         vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/vault"
+      environment:
+        https_proxy: ''
diff --git a/etc/kayobe/ansible/vault-generate-backend-tls.yml b/etc/kayobe/ansible/vault-generate-backend-tls.yml
@@ -18,7 +18,7 @@
 - name: Generate backend API certificates
   hosts: controllers:network
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     vault_intermediate_ca_name: "OS-TLS-INT"
   tasks:
     - name: Set a fact about the virtualenv on the remote system
@@ -53,6 +53,8 @@
         extra_params:
           ip_sans: "{{ internal_net_name | net_ip }}"
       register: backend_cert
+      environment:
+        https_proxy: ''
 
     - name: Ensure certificates directory exists
       file:
diff --git a/etc/kayobe/ansible/vault-generate-internal-tls.yml b/etc/kayobe/ansible/vault-generate-internal-tls.yml
@@ -3,7 +3,7 @@
   hosts: controllers
   run_once: true
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     vault_intermediate_ca_name: "OS-TLS-INT"
   tasks:
     - name: Include Vault keys
@@ -22,6 +22,8 @@
         extra_params:
           ip_sans: "{{ kolla_internal_vip_address }}"
       register: internal_cert
+      environment:
+        https_proxy: ''
 
     - name: Ensure certificates directory exists
       file:
diff --git a/etc/kayobe/ansible/vault-generate-test-external-tls.yml b/etc/kayobe/ansible/vault-generate-test-external-tls.yml
@@ -3,7 +3,7 @@
   hosts: controllers
   run_once: true
   vars:
-    vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200"
+    vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
     # NOTE: Using the same CA as internal TLS.
     vault_intermediate_ca_name: "OS-TLS-INT"
   tasks:
@@ -23,6 +23,8 @@
         extra_params:
           ip_sans: "{{ kolla_external_vip_address }}"
       register: external_cert
+      environment:
+        https_proxy: ''
 
     - name: Ensure certificates directory exists
       file:
diff --git a/etc/kayobe/ansible/vault-unseal-overcloud.yml b/etc/kayobe/ansible/vault-unseal-overcloud.yml
@@ -28,6 +28,9 @@
     - import_role:
         name: stackhpc.hashicorp.vault_unseal
       vars:
-        # NOTE: Need to unseal each backend, so don't use the VIP
-        vault_api_addr: "http://localhost:8200"
+        vault_api_addr: "https://{{ internal_net_name | net_ip }}:8200"
+        vault_unseal_token: "{{ vault_keys.root_token }}"
+        vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
         vault_unseal_keys: "{{ vault_keys.keys_base64 }}"
+      environment:
+        https_proxy: ''
diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml
@@ -8,12 +8,8 @@ kolla_enable_designate: true
 kolla_enable_redis: true
 kolla_enable_barbican: true
 
-# The multinode environment supports backend, external and internal TLS , but
-# it must be enabled in the correct order. See
-# https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-yoga/configuration/vault.html
-# for details.
-# kolla_enable_tls_external: true
-# kolla_enable_tls_internal: true
+kolla_enable_tls_external: true
+kolla_enable_tls_internal: true
 
 kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
 kolla_admin_openrc_cacert: "{{ kolla_public_openrc_cacert }}"
diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals-tls-config.yml b/etc/kayobe/environments/ci-multinode/kolla/globals-tls-config.yml
diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
diff --git a/releasenotes/notes/vault-without-haproxy-db55707e1cc53896.yaml b/releasenotes/notes/vault-without-haproxy-db55707e1cc53896.yaml
