Merge branch 'stackhpc/2023.1' into bugfix/INFRA-629-2

Author: dougszumski

.github/workflows/stackhpc-all-in-one.yml

@@ -167,7 +167,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
-          VM_VOLUME_SIZE: ${{ inputs.upgrade && '45' || '35' }}
+          VM_VOLUME_SIZE: ${{ inputs.upgrade && '50' || '35' }}
           VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
@@ -179,6 +179,7 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
       - name: Terraform Apply
+        id: tf_apply
         run: |
           for attempt in $(seq 5); do
               if terraform apply -auto-approve; then
@@ -355,6 +356,7 @@ jobs:
         if: inputs.upgrade
 
       - name: Tempest tests
+        id: tempest
         run: |
           mkdir -p tempest-artifacts
           docker run -t --rm \
@@ -366,16 +368,55 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: StackHPC OpenStack tests
+        id: stackhpc-openstack-tests
+        continue-on-error: true
+        run: |
+          mkdir -p sot-results
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -v $(pwd)/sot-results:/stack/sot-results \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-openstack-tests.yml'
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
+      - name: Collect diagnostic information
+        id: diagnostics
+        run: |
+          mkdir -p diagnostics
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -v $(pwd)/diagnostics:/stack/diagnostics \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml'
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }}
+
       - name: Upload test result artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' }}
-          path: tempest-artifacts/*
+          name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }}
+          path: |
+            diagnostics/
+            tempest-artifacts/
+            sot-results/
+        if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
 
       - name: Fail if any Tempest tests failed
         run: |
           test $(wc -l < tempest-artifacts/failed-tests) -lt 1
 
+      - name: Fail if any StackHPC OpenStack tests failed
+        run: |
+          echo "Some StackHPC OpenStack tests failed."
+          echo "See HTML results artifact (sot-results) for details."
+          exit 1
+        if: steps.stackhpc-openstack-tests.outcome == 'failure'
+
       - name: Destroy
         run: terraform destroy -auto-approve
         working-directory: ${{ github.workspace }}/terraform/aio

.github/workflows/stackhpc-container-image-build.yml

@@ -136,6 +136,10 @@ jobs:
         run: |
           curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
 
+      - name: Install yq
+        run: |
+          curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64.tar.gz | tar xz && sudo mv yq_linux_amd64 /usr/bin/yq
+
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -149,7 +153,7 @@ jobs:
       # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          sudo pip install docker
+          sudo pip install docker 'requests<2.32.0'
       
       - name: Get Kolla tag
         id: write-kolla-tag
@@ -253,7 +257,7 @@ jobs:
               if docker push $image; then
                 echo "Pushed $image"
                 break
-              elif $i == 5; then
+              elif [ $i -eq 5 ] ; then
                 echo "Failed to push $image"
                 echo $image >> image-build-logs/push-failed-images.txt
               else

doc/source/configuration/cephadm.rst

@@ -1,9 +1,9 @@
-================
-Cephadm & Kayobe
-================
+====
+Ceph
+====
 
 This section describes how to use the Cephadm integration included in StackHPC
-Kayobe configuration since Xena to deploy Ceph.
+Kayobe configuration to deploy Ceph.
 
 The Cephadm integration takes the form of custom playbooks that wrap
 around the Ansible `stackhpc.cephadm collection
@@ -19,10 +19,10 @@ create or modify Ceph cluster deployments. Supported features are:
 Resources
 =========
 
--  https://docs.ceph.com/en/pacific/cephadm/index.html
--  https://docs.ceph.com/en/pacific/
 -  https://docs.ceph.com/en/quincy/cephadm/index.html
 -  https://docs.ceph.com/en/quincy/
+-  https://docs.ceph.com/en/reef/cephadm/index.html
+-  https://docs.ceph.com/en/reef/
 -  https://github.com/stackhpc/ansible-collection-cephadm
 
 Configuration
@@ -107,7 +107,7 @@ OSD specification
 ~~~~~~~~~~~~~~~~~
 
 The following example is a basic OSD spec that adds OSDs for all
-available disks:
+available disks with encryption at rest:
 
 .. code:: yaml
 
@@ -118,9 +118,10 @@ available disks:
        host_pattern: "*"
      data_devices:
        all: true
+     encrypted: true
 
 More information about OSD service placement is available
-`here <https://docs.ceph.com/en/pacific/cephadm/services/osd/#advanced-osd-service-specifications>`__.
+`here <https://docs.ceph.com/en/quincy/cephadm/services/osd/#advanced-osd-service-specifications>`__.
 
 Container image
 ~~~~~~~~~~~~~~~
@@ -264,6 +265,24 @@ post-deployment configuration is applied. Commands in the
 ``cephadm_commands_post`` list are executed after the rest of the Ceph
 post-deployment configuration is applied.
 
+Messenger v2 encryption in transit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Messenger v2 is the default on-wire protocol since the Nautilus release. It
+supports `encryption of data in transit
+<https://docs.ceph.com/en/quincy/rados/configuration/msgr2/#connection-mode-configuration-options>`_,
+but this is not used by default. It may be enabled as follows:
+
+.. code:: yaml
+
+   # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands
+   # for format.
+   cephadm_commands_pre:
+    # Enable messenger v2 encryption in transit.
+    - "config set global ms_cluster_mode secure"
+    - "config set global ms_service_mode secure"
+    - "config set global ms_client_mode secure"
+
 Manila & CephFS
 ~~~~~~~~~~~~~~~
 

doc/source/configuration/monitoring.rst

@@ -126,6 +126,8 @@ depending on your configuration, you may need set the
 ``kolla_enable_prometheus_ceph_mgr_exporter`` variable to ``true`` in order to
 enable the ceph mgr exporter.
 
+.. _os-capacity:
+
 OpenStack Capacity
 ==================
 
@@ -149,9 +151,19 @@ project domain name in ``stackhpc-monitoring.yml``:
     stackhpc_os_capacity_openstack_region_name: <openstack_region_name>
 
 Additionally, you should ensure these credentials have the correct permissions
-for the exporter. If you are deploying in a cloud with internal TLS, you may be required
-to disable certificate verification for the OpenStack Capacity exporter
-if your certificate is not signed by a trusted CA.
+for the exporter.
+
+If you are deploying in a cloud with internal TLS, you may be required
+to provide a CA certificate for the OpenStack Capacity exporter if your
+certificate is not signed by a trusted CA. For example, to use a CA certificate
+named ``vault.crt`` that is also added to the Kolla containers:
+
+.. code-block:: yaml
+
+    stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt"
+
+Alternatively, to disable certificate verification for the OpenStack Capacity
+exporter:
 
 .. code-block:: yaml
 

doc/source/configuration/release-train.rst

@@ -147,6 +147,33 @@ By default, HashiCorp images (Consul and Vault) are not synced from Docker Hub
 to the local Pulp. To sync these images, set ``stackhpc_sync_hashicorp_images``
 to ``true``.
 
+Custom container images
+-----------------------
+
+A custom list of container images can be synced to the local Pulp using the
+``stackhpc_pulp_repository_container_repos_extra`` and
+``stackhpc_pulp_distribution_container_extra`` variables.
+
+.. code-block:: yaml
+
+   # List of extra container image repositories.
+   stackhpc_pulp_repository_container_repos_extra:
+     - name: "certbot/certbot"
+       url: "https://registry-1.docker.io"
+       policy: on_demand
+       proxy_url: "{{ pulp_proxy_url }}"
+       state: present
+       include_tags: "nightly"
+       required: True
+
+   # List of extra container image distributions.
+   stackhpc_pulp_distribution_container_extra:
+     - name: certbot
+       repository: certbot/certbot
+       base_path: certbot/certbot
+       state: present
+       required: True
+
 Usage
 =====
 

doc/source/configuration/vault.rst

@@ -84,47 +84,6 @@ Setup Vault on the seed node
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud.key
 
-Setup HAProxy config for Vault
-------------------------------
-
-1. Create the HAProxy config to reverse proxy the Vault HA container
-
-   Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes.
-
-   Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg
-
-   .. code-block::
-
-      # Delete "verify none" if not using self-signed/unknown issuer
-      {% raw %}
-      frontend vault_front
-         mode tcp
-         option tcplog
-         bind {{ kolla_internal_vip_address }}:8200
-         default_backend vault_back
-
-      backend vault_back
-         mode tcp
-         option httpchk GET /v1/sys/health
-         # https://www.vaultproject.io/api-docs/system/health
-         # 200: initialized, unsealed, and active
-         # 501: not initialised (required for bootstrapping)
-         # 503: sealed (required for bootstrapping)
-         http-check expect rstatus (200|501|503)
-
-      {% for host in groups['control'] %}
-      {% set host_name = hostvars[host].ansible_facts.hostname %}
-      {% set host_ip = 'api' | kolla_address(host) %}
-         server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5
-      {% endfor %}
-      {% endraw %}
-
-2. Deploy HAProxy with the new Vault service configuration:
-
-   .. code-block::
-
-      kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
-
 Setup Vault HA on the overcloud hosts
 -------------------------------------
 
@@ -215,6 +174,55 @@ Create the backend TLS and RabbitMQ TLS certificates
 
       ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/<controller>-key.pem
 
+.. _vault-haproxy:
+
+HAProxy integration
+===================
+
+It is possible to expose the overcloud Vault service via the Kolla Ansible HAProxy load balancer.
+This provides a single highly available API endpoint, as well as monitoring of the Vault backends when combined with Prometheus.
+HAProxy integration is no longer required for generating OpenStack control plane certificates, making it possible to deploy Vault and generate certificates before any containers have been deployed by Kolla Ansible.
+
+1. Create the HAProxy config to reverse proxy the Vault HA container
+
+   Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes.
+
+   Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg
+
+   .. code-block::
+
+      # Delete "verify none" if not using self-signed/unknown issuer
+      {% raw %}
+      frontend vault_front
+         mode tcp
+         option tcplog
+         bind {{ kolla_internal_vip_address }}:8200
+         default_backend vault_back
+
+      backend vault_back
+         mode tcp
+         option httpchk GET /v1/sys/health
+         # https://www.vaultproject.io/api-docs/system/health
+         # 200: initialized, unsealed, and active
+         # 429: standby
+         http-check expect rstatus (200|429)
+
+      {% for host in groups['control'] %}
+      {% set host_name = hostvars[host].ansible_facts.hostname %}
+      {% set host_ip = 'api' | kolla_address(host) %}
+         server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5
+      {% endfor %}
+      {% endraw %}
+
+2. If HAProxy has not yet been deployed, continue to :ref:`certificates deployment <vault-certificates>`.
+   If HAProxy has been deployed, it may be redeployed with the new Vault service configuration:
+
+   .. code-block::
+
+      kayobe overcloud service deploy -kt haproxy
+
+.. _vault-certificates:
+
 Certificates deployment
 =======================
 
@@ -231,6 +239,7 @@ Enable the required TLS variables in kayobe and kolla
 
       # Whether TLS is enabled for the external API endpoints. Default is 'no'.
       kolla_enable_tls_external: yes
+      kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
 
    See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest.
 
@@ -240,6 +249,9 @@ Enable the required TLS variables in kayobe and kolla
 
       # Whether TLS is enabled for the internal API endpoints. Default is 'no'.
       kolla_enable_tls_internal: yes
+      kolla_admin_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}"
+
+   See :ref:`os-capacity` for information on adding CA certificates to the trust store when deploying the OpenStack Capacity exporter.
 
 3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml
 
@@ -289,6 +301,8 @@ Enable the required TLS variables in kayobe and kolla
 Barbican integration
 ====================
 
+Barbican integration depends on :ref:`HAProxy integration <vault-haproxy>`.
+
 Enable Barbican in kayobe
 -------------------------
 
@@ -339,7 +353,7 @@ Configure Barbican
       enabled_secretstore_plugins=vault_plugin
 
       [vault_plugin]
-      vault_url = https://{{ kolla_internal_vip_address }}:8200
+      vault_url = https://{{ kolla_internal_fqdn }}:8200
       use_ssl = True
       {% raw %}
       ssl_ca_crt_file = {{ openstack_cacert }}