diff --git a/build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml b/build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml index d1e9faf222..4871c6bb0b 100644 --- a/build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml +++ b/build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml @@ -13594,6 +13594,53 @@ spec: type: boolean extensions: properties: + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' pgAudit: type: boolean pgRepack: @@ -13605,6 +13652,11 @@ spec: pgvector: type: boolean type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: |- The image name to use for PostgreSQL containers. When omitted, the value @@ -30978,6 +31030,10 @@ spec: description: The PostgreSQL system identifier reported by Patroni. type: string type: object + pgTDERevision: + description: Identifies the pg_tde configuration that have been installed + into PostgreSQL. + type: string pgbackrest: description: Status information for pgBackRest properties: diff --git a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml index 9132f12648..b8502bcc26 100644 --- a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml +++ b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml @@ -13696,11 +13696,11 @@ spec: description: The specification of extensions. properties: builtin: + description: 'Deprecated: Use extensions. instead. + This field will be removed after 3.4.0.' properties: pg_audit: type: boolean - pg_cron: - type: boolean pg_repack: type: boolean pg_stat_monitor: @@ -13709,8 +13709,6 @@ spec: type: boolean pgvector: type: boolean - set_user: - type: boolean type: object custom: items: @@ -13729,6 +13727,88 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + pg_audit: + properties: + enabled: + type: boolean + type: object + pg_cron: + properties: + enabled: + type: boolean + type: object + pg_repack: + properties: + enabled: + type: boolean + type: object + pg_stat_monitor: + properties: + enabled: + type: boolean + type: object + pg_stat_statements: + properties: + enabled: + type: boolean + type: object + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' + pgvector: + properties: + enabled: + type: boolean + type: object + set_user: + properties: + enabled: + type: boolean + type: object storage: properties: bucket: @@ -13811,6 +13891,11 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: The image name to use for PostgreSQL containers. type: string @@ -28842,6 +28927,9 @@ spec: - postgresVersion type: object x-kubernetes-validations: + - message: pg_tde is only supported for PG17 and above + rule: '!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) + || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17' - message: PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, diff --git a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml index c0a165b300..4c0846933d 100644 --- a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml +++ b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml @@ -14394,11 +14394,11 @@ spec: description: The specification of extensions. properties: builtin: + description: 'Deprecated: Use extensions. instead. + This field will be removed after 3.4.0.' properties: pg_audit: type: boolean - pg_cron: - type: boolean pg_repack: type: boolean pg_stat_monitor: @@ -14407,8 +14407,6 @@ spec: type: boolean pgvector: type: boolean - set_user: - type: boolean type: object custom: items: @@ -14427,6 +14425,88 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + pg_audit: + properties: + enabled: + type: boolean + type: object + pg_cron: + properties: + enabled: + type: boolean + type: object + pg_repack: + properties: + enabled: + type: boolean + type: object + pg_stat_monitor: + properties: + enabled: + type: boolean + type: object + pg_stat_statements: + properties: + enabled: + type: boolean + type: object + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' + pgvector: + properties: + enabled: + type: boolean + type: object + set_user: + properties: + enabled: + type: boolean + type: object storage: properties: bucket: @@ -14509,6 +14589,11 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: The image name to use for PostgreSQL containers. type: string @@ -29540,6 +29625,9 @@ spec: - postgresVersion type: object x-kubernetes-validations: + - message: pg_tde is only supported for PG17 and above + rule: '!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) + || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17' - message: PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, diff --git a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml index aef40e758b..bcce3a5c0c 100644 --- a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml +++ b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml @@ -13551,6 +13551,53 @@ spec: type: boolean extensions: properties: + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' pgAudit: type: boolean pgCron: @@ -13566,6 +13613,11 @@ spec: setUser: type: boolean type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: |- The image name to use for PostgreSQL containers. When omitted, the value @@ -30928,6 +30980,10 @@ spec: description: The PostgreSQL system identifier reported by Patroni. type: string type: object + pgTDERevision: + description: Identifies the pg_tde configuration that have been installed + into PostgreSQL. + type: string pgbackrest: description: Status information for pgBackRest properties: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 034da545d9..3ec5480904 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -14691,11 +14691,11 @@ spec: description: The specification of extensions. properties: builtin: + description: 'Deprecated: Use extensions. instead. + This field will be removed after 3.4.0.' properties: pg_audit: type: boolean - pg_cron: - type: boolean pg_repack: type: boolean pg_stat_monitor: @@ -14704,8 +14704,6 @@ spec: type: boolean pgvector: type: boolean - set_user: - type: boolean type: object custom: items: @@ -14724,6 +14722,88 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + pg_audit: + properties: + enabled: + type: boolean + type: object + pg_cron: + properties: + enabled: + type: boolean + type: object + pg_repack: + properties: + enabled: + type: boolean + type: object + pg_stat_monitor: + properties: + enabled: + type: boolean + type: object + pg_stat_statements: + properties: + enabled: + type: boolean + type: object + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' + pgvector: + properties: + enabled: + type: boolean + type: object + set_user: + properties: + enabled: + type: boolean + type: object storage: properties: bucket: @@ -14806,6 +14886,11 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: The image name to use for PostgreSQL containers. type: string @@ -29837,6 +29922,9 @@ spec: - postgresVersion type: object x-kubernetes-validations: + - message: pg_tde is only supported for PG17 and above + rule: '!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) + || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17' - message: PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, @@ -51688,6 +51776,53 @@ spec: type: boolean extensions: properties: + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' pgAudit: type: boolean pgCron: @@ -51703,6 +51838,11 @@ spec: setUser: type: boolean type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: |- The image name to use for PostgreSQL containers. When omitted, the value @@ -69065,6 +69205,10 @@ spec: description: The PostgreSQL system identifier reported by Patroni. type: string type: object + pgTDERevision: + description: Identifies the pg_tde configuration that have been installed + into PostgreSQL. + type: string pgbackrest: description: Status information for pgBackRest properties: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 5099899f9a..d015200e7c 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -774,14 +774,31 @@ spec: # disableSSL: false # secret: # name: cluster1-extensions-secret -# builtin: -# pg_stat_monitor: false -# pg_stat_statements: false -# pg_audit: true -# pgvector: false -# pg_repack: false -# pg_cron: false -# set_user: false +# pg_stat_monitor: +# enabled: false +# pg_stat_statements: +# enabled: false +# pg_audit: +# enabled: true +# pgvector: +# enabled: false +# pg_repack: +# enabled: false +# pg_cron: +# enabled: false +# set_user: +# enabled: false +# pg_tde: +# enabled: false +# vault: +# host: https://vault-service:8200 +# mountPath: tde +# tokenSecret: +# name: pg-tde-vault-secret +# key: token +# caSecret: +# name: pg-tde-vault-secret +# key: ca.crt # custom: # - name: pg_cron # version: 1.6.1 diff --git a/deploy/crd.yaml b/deploy/crd.yaml index f2cb618879..dc13656b00 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -14691,11 +14691,11 @@ spec: description: The specification of extensions. properties: builtin: + description: 'Deprecated: Use extensions. instead. + This field will be removed after 3.4.0.' properties: pg_audit: type: boolean - pg_cron: - type: boolean pg_repack: type: boolean pg_stat_monitor: @@ -14704,8 +14704,6 @@ spec: type: boolean pgvector: type: boolean - set_user: - type: boolean type: object custom: items: @@ -14724,6 +14722,88 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + pg_audit: + properties: + enabled: + type: boolean + type: object + pg_cron: + properties: + enabled: + type: boolean + type: object + pg_repack: + properties: + enabled: + type: boolean + type: object + pg_stat_monitor: + properties: + enabled: + type: boolean + type: object + pg_stat_statements: + properties: + enabled: + type: boolean + type: object + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' + pgvector: + properties: + enabled: + type: boolean + type: object + set_user: + properties: + enabled: + type: boolean + type: object storage: properties: bucket: @@ -14806,6 +14886,11 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: The image name to use for PostgreSQL containers. type: string @@ -29837,6 +29922,9 @@ spec: - postgresVersion type: object x-kubernetes-validations: + - message: pg_tde is only supported for PG17 and above + rule: '!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) + || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17' - message: PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, @@ -51688,6 +51776,53 @@ spec: type: boolean extensions: properties: + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' pgAudit: type: boolean pgCron: @@ -51703,6 +51838,11 @@ spec: setUser: type: boolean type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: |- The image name to use for PostgreSQL containers. When omitted, the value @@ -69065,6 +69205,10 @@ spec: description: The PostgreSQL system identifier reported by Patroni. type: string type: object + pgTDERevision: + description: Identifies the pg_tde configuration that have been installed + into PostgreSQL. + type: string pgbackrest: description: Status information for pgBackRest properties: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 2b67891aa7..d32403cc3c 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -14691,11 +14691,11 @@ spec: description: The specification of extensions. properties: builtin: + description: 'Deprecated: Use extensions. instead. + This field will be removed after 3.4.0.' properties: pg_audit: type: boolean - pg_cron: - type: boolean pg_repack: type: boolean pg_stat_monitor: @@ -14704,8 +14704,6 @@ spec: type: boolean pgvector: type: boolean - set_user: - type: boolean type: object custom: items: @@ -14724,6 +14722,88 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + pg_audit: + properties: + enabled: + type: boolean + type: object + pg_cron: + properties: + enabled: + type: boolean + type: object + pg_repack: + properties: + enabled: + type: boolean + type: object + pg_stat_monitor: + properties: + enabled: + type: boolean + type: object + pg_stat_statements: + properties: + enabled: + type: boolean + type: object + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' + pgvector: + properties: + enabled: + type: boolean + type: object + set_user: + properties: + enabled: + type: boolean + type: object storage: properties: bucket: @@ -14806,6 +14886,11 @@ spec: type: string type: object type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: The image name to use for PostgreSQL containers. type: string @@ -29837,6 +29922,9 @@ spec: - postgresVersion type: object x-kubernetes-validations: + - message: pg_tde is only supported for PG17 and above + rule: '!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) + || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17' - message: PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, @@ -51688,6 +51776,53 @@ spec: type: boolean extensions: properties: + pg_tde: + properties: + enabled: + type: boolean + vault: + properties: + caSecret: + description: Name of the secret that contains the CA certificate + for SSL verification. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + host: + description: Host of Vault server. + type: string + mountPath: + default: secret/data + description: The mount point on the Vault server where + the key provider should store the keys. + type: string + tokenSecret: + description: Name of the secret that contains the access + token with read and write access to the mount path. + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - host + - tokenSecret + type: object + type: object + x-kubernetes-validations: + - message: vault is required for enabling pg_tde + rule: '!has(self.enabled) || (has(self.enabled) && self.enabled + == false) || has(self.vault)' pgAudit: type: boolean pgCron: @@ -51703,6 +51838,11 @@ spec: setUser: type: boolean type: object + x-kubernetes-validations: + - message: to disable pg_tde first set enabled=false without removing + vault and wait for pod restarts + rule: '!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) + || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)' image: description: |- The image name to use for PostgreSQL containers. When omitted, the value @@ -69065,6 +69205,10 @@ spec: description: The PostgreSQL system identifier reported by Patroni. type: string type: object + pgTDERevision: + description: Identifies the pg_tde configuration that have been installed + into PostgreSQL. + type: string pgbackrest: description: Status information for pgBackRest properties: diff --git a/e2e-tests/functions b/e2e-tests/functions index febfb71c00..647a04761c 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -442,12 +442,29 @@ run_psql() { bash -c "printf '$command\n' | PGPASSWORD="\'$password\'" psql -v ON_ERROR_STOP=1 -t -q $uri" } +run_psql_command() { + local command=${1} + local uri=${2} + local driver=${3:-postgres} + + kubectl -n ${NAMESPACE} exec $(get_client_pod) -- \ + psql -v ON_ERROR_STOP=1 -t -q "${driver}://${uri}" -c "${command}" +} + get_psql_user_pass() { local secret_name=${1} kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.password | base64decode}}' } +get_psql_uri() { + local cluster=$1 + local user=$2 + local secret_name="${cluster}-pguser-${user}" + + echo "${user}:$(get_psql_user_pass ${secret_name})@$(get_psql_user_host ${secret_name})" +} + get_pgbouncer_host() { local secret_name=${1} @@ -2015,3 +2032,186 @@ verify_k8s_nodes_version() { fi done } + +function vault_tls() { + local name=${1:-vault-service} + local tmp_dir=$2 + + local service=$name + local namespace=$name + local secret_name=$name + local csr_name=vault-csr-${RANDOM} + local csr_api_ver="v1" + local csr_signer + local platform=$(detect_k8s_platform) + + echo "Detected platform: ${platform}" + + case ${platform} in + eks) + csr_signer=" signerName: beta.eks.amazonaws.com/app-serving" + ;; + *) + csr_signer=" signerName: kubernetes.io/kubelet-serving" + ;; + esac + + openssl genrsa -out ${tmp_dir}/vault.key 2048 + cat <${tmp_dir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +DNS.4 = ${service}.${namespace}.svc.cluster.local +IP.1 = 127.0.0.1 +EOF + + openssl req -new -key ${tmp_dir}/vault.key -subj "/CN=system:node:${service}.${namespace}.svc;/O=system:nodes" -out ${tmp_dir}/server.csr -config ${tmp_dir}/csr.conf + + cat <${tmp_dir}/csr.yaml +apiVersion: certificates.k8s.io/${csr_api_ver} +kind: CertificateSigningRequest +metadata: + name: ${csr_name} +spec: + groups: + - system:authenticated + request: $(cat ${tmp_dir}/server.csr | base64 | tr -d '\n') +${csr_signer} + usages: + - digital signature + - key encipherment + - server auth +EOF + + kubectl create -f ${tmp_dir}/csr.yaml + sleep 10 + kubectl certificate approve ${csr_name} + kubectl get csr ${csr_name} -o jsonpath='{.status.certificate}' >${tmp_dir}/serverCert + openssl base64 -in ${tmp_dir}/serverCert -d -A -out ${tmp_dir}/vault.crt + kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d >${tmp_dir}/vault.ca + if [[ -n ${OPENSHIFT} ]]; then + if [[ "x$(kubectl get namespaces | awk '{print $1}' | grep openshift-kube-controller-manager-operator)" != "x" ]]; then + #Detecting openshift 4+ + kubectl -n openshift-kube-controller-manager-operator get secret csr-signer -o jsonpath='{.data.tls\.crt}' \ + | base64 -d >${tmp_dir}/vault.ca + else + local ca_secret_name=$(kubectl -n default get secrets \ + | grep default \ + | grep service-account-token \ + | head -n 1 \ + | awk {'print $1'}) + kubectl -n default get secret ${ca_secret_name} -o jsonpath='{.data.ca\.crt}' \ + | base64 -d >${tmp_dir}/vault.ca + fi + fi + kubectl create secret generic ${secret_name} \ + --namespace ${namespace} \ + --from-file=vault.key=${tmp_dir}/vault.key \ + --from-file=vault.crt=${tmp_dir}/vault.crt \ + --from-file=vault.ca=${tmp_dir}/vault.ca +} + +function start_vault() { + local name=${1:-vault-service} + local protocol=${2:-http} + local platform=kubernetes + local tmp_dir=$(mktemp -d) + + if [[ -n ${OPENSHIFT} ]]; then + platform=openshift + oc patch clusterrole system:auth-delegator --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]' + local extra_args="--set server.image.repository=docker.io/hashicorp/vault --set injector.image.repository=docker.io/hashicorp/vault-k8s" + fi + + create_namespace "$name" "skip_clean" + helm repo add hashicorp https://helm.releases.hashicorp.com + helm uninstall "$name" || : + + echo "install Vault $name" + + if [ $protocol == "https" ]; then + vault_tls "${name}" ${tmp_dir} + helm install $name hashicorp/vault \ + --disable-openapi-validation \ + --version $VAULT_VER \ + --namespace "$name" \ + --set global.tlsDisable=false \ + --set global.platform="${platform}" \ + --set server.dataStorage.enabled=false \ + --set server.standalone.enabled=true \ + --set server.ha.raft.enabled=false \ + --set server.extraVolumes[0].type=secret \ + --set server.extraVolumes[0].name=$name \ + --set server.extraEnvironmentVars.VAULT_CACERT=/vault/userconfig/$name/vault.ca \ + $extra_args \ + --set server.standalone.config=" \ +listener \"tcp\" { + address = \"[::]:8200\" + cluster_address = \"[::]:8201\" + tls_cert_file = \"/vault/userconfig/$name/vault.crt\" + tls_key_file = \"/vault/userconfig/$name/vault.key\" + tls_client_ca_file = \"/vault/userconfig/$name/vault.ca\" +} + +storage \"file\" { + path = \"/vault/data\" +}" + + else + helm install $name hashicorp/vault \ + --disable-openapi-validation \ + --version $VAULT_VER \ + --namespace "$name" \ + --set server.dataStorage.enabled=false \ + --set server.standalone.enabled=true \ + --set server.ha.raft.enabled=false \ + $extra_args \ + --set global.platform="${platform}" + fi + + if [[ -n ${OPENSHIFT} ]]; then + oc patch clusterrole $name-agent-injector-clusterrole --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]' + oc adm policy add-scc-to-user privileged $name-agent-injector + fi + + set +o xtrace + local retry=0 + echo -n pod/$name-0 + until kubectl -n ${name} get pod/$name-0 -o 'jsonpath={.status.containerStatuses[0].state}' 2>/dev/null | grep 'running'; do + echo -n . + sleep 1 + let retry+=1 + if [ "$retry" -ge 480 ]; then + kubectl -n ${name} describe pod/$name-0 + kubectl -n ${name} logs $name-0 + echo max retry count "$retry" reached. something went wrong with vault + exit 1 + fi + done + + kubectl -n ${name} exec -it $name-0 -- vault operator init -tls-skip-verify -key-shares=1 -key-threshold=1 -format=json >"$tmp_dir/$name" + local unsealKey=$(jq -r ".unseal_keys_b64[]" <"$tmp_dir/$name") + local token=$(jq -r ".root_token" <"$tmp_dir/$name") + sleep 10 + + kubectl -n ${name} exec -it $name-0 -- vault operator unseal -tls-skip-verify "$unsealKey" + kubectl -n ${name} exec -it $name-0 -- \ + sh -c "export VAULT_TOKEN=$token && export VAULT_LOG_LEVEL=trace \ + && vault secrets enable --version=2 -path=tde kv \ + && vault audit enable file file_path=/vault/vault-audit.log" + sleep 10 + + kubectl -n "${NAMESPACE}" create secret generic vault-secret \ + --from-literal=token=${token} \ + --from-file=ca.crt=${tmp_dir}/vault.ca +} diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 91472001a7..4149255f08 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -21,6 +21,7 @@ monitoring-pmm3 one-pod operator-self-healing pgbouncer-mtls +pg-tde pitr scaling scheduled-backup diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 787d2e4ef1..f7ddc26964 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -26,6 +26,7 @@ monitoring-pmm3 one-pod operator-self-healing pgbouncer-mtls +pg-tde pitr scaling scheduled-backup diff --git a/e2e-tests/tests/builtin-extensions/00-deploy-operator.yaml b/e2e-tests/tests/builtin-extensions/00-deploy-operator.yaml index 96329aabb8..ae4a2419aa 100644 --- a/e2e-tests/tests/builtin-extensions/00-deploy-operator.yaml +++ b/e2e-tests/tests/builtin-extensions/00-deploy-operator.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -13,3 +12,4 @@ commands: deploy_client deploy_s3_secrets deploy_minio + timeout: 120 diff --git a/e2e-tests/tests/builtin-extensions/03-install-all-ext.yaml b/e2e-tests/tests/builtin-extensions/03-install-all-ext.yaml index c9fb037b2a..ef16706494 100644 --- a/e2e-tests/tests/builtin-extensions/03-install-all-ext.yaml +++ b/e2e-tests/tests/builtin-extensions/03-install-all-ext.yaml @@ -11,11 +11,15 @@ spec: pgaudit.log_level: 'warning' logging_collector: 'off' extensions: - builtin: - pg_stat_monitor: false # you can't enable both pg_stat_statements and pg_stat_monitor - pg_stat_statements: true - pg_audit: true - pgvector: true - pg_repack: true - pg_cron: true - set_user: true + pg_stat_statements: + enabled: true + pg_audit: + enabled: true + pgvector: + enabled: true + pg_repack: + enabled: true + pg_cron: + enabled: true + set_user: + enabled: true diff --git a/e2e-tests/tests/builtin-extensions/04-check-extensions.yaml b/e2e-tests/tests/builtin-extensions/04-check-extensions.yaml index 8b06573068..dd65480cc4 100644 --- a/e2e-tests/tests/builtin-extensions/04-check-extensions.yaml +++ b/e2e-tests/tests/builtin-extensions/04-check-extensions.yaml @@ -1,8 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 180 commands: - - script: |- + - timeout: 180 + script: |- set -o errexit set -o xtrace @@ -12,7 +12,10 @@ commands: data=$(kubectl -n ${NAMESPACE} exec $(get_client_pod) -- psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:$(get_psql_user_pass builtin-extensions-pguser-postgres)@$(get_psql_user_host builtin-extensions-pguser-postgres) -c "\c postgres" -c "SELECT extname FROM pg_extension ORDER BY extname") for ext in pg_cron pg_repack pg_stat_statements pgaudit plpgsql set_user vector; do - echo "${data}" | grep -q "${ext}" || return 1 + if ! echo "${data}" | grep -q "${ext}"; then + echo "${ext} is not enabled!" + return 1 + fi done } diff --git a/e2e-tests/tests/builtin-extensions/06-enable-pgstatmonitor.yaml b/e2e-tests/tests/builtin-extensions/06-enable-pgstatmonitor.yaml index 88d6a82027..d5f1a53be2 100644 --- a/e2e-tests/tests/builtin-extensions/06-enable-pgstatmonitor.yaml +++ b/e2e-tests/tests/builtin-extensions/06-enable-pgstatmonitor.yaml @@ -26,14 +26,26 @@ commands: } }, \"extensions\": { - \"builtin\": { - \"pg_stat_monitor\": ${PG_STAT_MONITOR}, - \"pg_stat_statements\": false, - \"pg_audit\": true, - \"pgvector\": true, - \"pg_repack\": true, - \"pg_cron\": true, - \"set_user\": true + \"pg_stat_monitor\": { + \"enabled\": ${PG_STAT_MONITOR} + }, + \"pg_stat_statements\": { + \"enabled\": false + }, + \"pg_audit\": { + \"enabled\": true + }, + \"pgvector\": { + \"enabled\": true + }, + \"pg_repack\": { + \"enabled\": true + }, + \"pg_cron\": { + \"enabled\": true + }, + \"set_user\": { + \"enabled\": true } } } diff --git a/e2e-tests/tests/builtin-extensions/09-uninstall-all-ext.yaml b/e2e-tests/tests/builtin-extensions/09-uninstall-all-ext.yaml index 827cde4fa5..bde24906e9 100644 --- a/e2e-tests/tests/builtin-extensions/09-uninstall-all-ext.yaml +++ b/e2e-tests/tests/builtin-extensions/09-uninstall-all-ext.yaml @@ -4,11 +4,17 @@ metadata: name: builtin-extensions spec: extensions: - builtin: - pg_stat_monitor: false - pg_stat_statements: false - pg_audit: false - pgvector: false - pg_repack: false - pg_cron: false - set_user: false + pg_stat_monitor: + enabled: false + pg_stat_statements: + enabled: false + pg_audit: + enabled: false + pgvector: + enabled: false + pg_repack: + enabled: false + pg_cron: + enabled: false + set_user: + enabled: false diff --git a/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml b/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml index 0cfe9bbd0e..38092cab34 100644 --- a/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml +++ b/e2e-tests/tests/custom-extensions/00-deploy-operator.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -14,3 +13,4 @@ commands: deploy_s3_secrets deploy_minio copy_custom_extensions_form_aws + timeout: 120 diff --git a/e2e-tests/tests/major-upgrade-17-to-18/00-deploy-operator.yaml b/e2e-tests/tests/major-upgrade-17-to-18/00-deploy-operator.yaml index 5d21c58f9d..742248d080 100644 --- a/e2e-tests/tests/major-upgrade-17-to-18/00-deploy-operator.yaml +++ b/e2e-tests/tests/major-upgrade-17-to-18/00-deploy-operator.yaml @@ -1,7 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: |- + - timeout: 600 + script: |- set -o errexit set -o xtrace @@ -11,3 +12,5 @@ commands: deploy_operator deploy_client deploy_s3_secrets + + start_vault vault-major-upgrade https diff --git a/e2e-tests/tests/major-upgrade-17-to-18/01-assert.yaml b/e2e-tests/tests/major-upgrade-17-to-18/01-assert.yaml index 6dcf7051e1..4e4ee00895 100644 --- a/e2e-tests/tests/major-upgrade-17-to-18/01-assert.yaml +++ b/e2e-tests/tests/major-upgrade-17-to-18/01-assert.yaml @@ -1,6 +1,12 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 120 +commands: + - script: | + tmp_file=$(mktemp) + kubectl -n $NAMESPACE get pg major-upgrade-17-to-18 -o yaml | yq '.status.conditions.[] | select(.type == "PGTDEEnabled")' > ${tmp_file} + [[ $(cat ${tmp_file} | yq .reason) == "Enabled" ]] + [[ $(cat ${tmp_file} | yq .status) == "True" ]] --- kind: StatefulSet apiVersion: apps/v1 diff --git a/e2e-tests/tests/major-upgrade-17-to-18/01-create-cluster.yaml b/e2e-tests/tests/major-upgrade-17-to-18/01-create-cluster.yaml index 817d6c619d..33a5b72827 100644 --- a/e2e-tests/tests/major-upgrade-17-to-18/01-create-cluster.yaml +++ b/e2e-tests/tests/major-upgrade-17-to-18/01-create-cluster.yaml @@ -21,5 +21,12 @@ commands: .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true | .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = \"pg_cron\" | - .spec.extensions.custom += [{\"name\": \"pg_cron\", \"version\": \"1.6.6\"}]" \ + .spec.extensions.custom += [{\"name\": \"pg_cron\", \"version\": \"1.6.6\"}] | + .spec.extensions.pg_tde.enabled = true | + .spec.extensions.pg_tde.vault.host = \"https://vault-major-upgrade.vault-major-upgrade.svc:8200\" | + .spec.extensions.pg_tde.vault.mountPath = \"tde\" | + .spec.extensions.pg_tde.vault.tokenSecret.name = \"vault-secret\" | + .spec.extensions.pg_tde.vault.tokenSecret.key = \"token\" | + .spec.extensions.pg_tde.vault.caSecret.name = \"vault-secret\" | + .spec.extensions.pg_tde.vault.caSecret.key = \"ca.crt\"" \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/major-upgrade-17-to-18/02-write-data.yaml b/e2e-tests/tests/major-upgrade-17-to-18/02-write-data.yaml index f2961b017f..faeb85d7fa 100644 --- a/e2e-tests/tests/major-upgrade-17-to-18/02-write-data.yaml +++ b/e2e-tests/tests/major-upgrade-17-to-18/02-write-data.yaml @@ -8,7 +8,7 @@ commands: source ../../functions run_psql_local \ - 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ + 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) USING tde_heap;' \ "postgres:$(get_psql_user_pass major-upgrade-17-to-18-pguser-postgres)@$(get_psql_user_host major-upgrade-17-to-18-pguser-postgres)" run_psql_local \ diff --git a/e2e-tests/tests/major-upgrade-17-to-18/05-assert.yaml b/e2e-tests/tests/major-upgrade-17-to-18/05-assert.yaml new file mode 100644 index 0000000000..1aa25a7be8 --- /dev/null +++ b/e2e-tests/tests/major-upgrade-17-to-18/05-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 05-verify-extension +data: + pg_tde_extension: ' pg_tde' +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 05-verify-encryption +data: + pg_tde_is_encrypted: ' t' diff --git a/e2e-tests/tests/major-upgrade-17-to-18/05-post-upgrade.yaml b/e2e-tests/tests/major-upgrade-17-to-18/05-post-upgrade.yaml index 7ae6ffafa1..52852b8f5f 100644 --- a/e2e-tests/tests/major-upgrade-17-to-18/05-post-upgrade.yaml +++ b/e2e-tests/tests/major-upgrade-17-to-18/05-post-upgrade.yaml @@ -11,3 +11,18 @@ commands: kubectl -n ${NAMESPACE} exec ${primary} -- vacuumdb --all --analyze-in-stages --missing-stats-only kubectl -n ${NAMESPACE} exec ${primary} -- vacuumdb --all --analyze-only kubectl -n ${NAMESPACE} exec ${primary} -- /pgdata/delete_old_cluster.sh + + result=$(run_psql_command \ + "SELECT extname FROM pg_extension WHERE extname = 'pg_tde';" \ + "$(get_psql_uri major-upgrade-17-to-18 postgres)") + kubectl -n "${NAMESPACE}" create configmap 05-verify-extension --from-literal=pg_tde_extension="$result" + + result=$(run_psql_command \ + "SELECT pg_tde_is_encrypted('myapp');" \ + "$(get_psql_uri major-upgrade-17-to-18 postgres)/myapp") + kubectl -n "${NAMESPACE}" create configmap 05-verify-encryption --from-literal=pg_tde_is_encrypted="$result" + + # pg_tde_verify_key will throw an error if it fails + run_psql_command \ + "SELECT pg_tde_verify_key();" \ + "$(get_psql_uri major-upgrade-17-to-18 postgres)/myapp" diff --git a/e2e-tests/tests/pg-tde/00-assert.yaml b/e2e-tests/tests/pg-tde/00-assert.yaml new file mode 100644 index 0000000000..ae5a062d84 --- /dev/null +++ b/e2e-tests/tests/pg-tde/00-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconapgclusters.pgv2.percona.com +spec: + group: pgv2.percona.com + names: + kind: PerconaPGCluster + listKind: PerconaPGClusterList + plural: perconapgclusters + singular: perconapgcluster + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/pg-tde/00-deploy-operator.yaml b/e2e-tests/tests/pg-tde/00-deploy-operator.yaml new file mode 100644 index 0000000000..1aaca58be2 --- /dev/null +++ b/e2e-tests/tests/pg-tde/00-deploy-operator.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + deploy_operator + deploy_client diff --git a/e2e-tests/tests/pg-tde/01-assert.yaml b/e2e-tests/tests/pg-tde/01-assert.yaml new file mode 100644 index 0000000000..432369127d --- /dev/null +++ b/e2e-tests/tests/pg-tde/01-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: v1 +kind: Secret +metadata: + name: vault-secret diff --git a/e2e-tests/tests/pg-tde/01-deploy-vault.yaml b/e2e-tests/tests/pg-tde/01-deploy-vault.yaml new file mode 100644 index 0000000000..d7127630f6 --- /dev/null +++ b/e2e-tests/tests/pg-tde/01-deploy-vault.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + start_vault vault-service https + timeout: 600 diff --git a/e2e-tests/tests/pg-tde/02-assert.yaml b/e2e-tests/tests/pg-tde/02-assert.yaml new file mode 100644 index 0000000000..2838f30d6f --- /dev/null +++ b/e2e-tests/tests/pg-tde/02-assert.yaml @@ -0,0 +1,118 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: | + tmp_file=$(mktemp) + kubectl -n $NAMESPACE get pg pg-tde -o yaml | yq '.status.conditions.[] | select(.type == "PGTDEEnabled")' > ${tmp_file} + [[ $(cat ${tmp_file} | yq .message) == "pg_tde is enabled in PerconaPGCluster" ]] + [[ $(cat ${tmp_file} | yq .observedGeneration) == "1" ]] + [[ $(cat ${tmp_file} | yq .reason) == "Enabled" ]] + [[ $(cat ${tmp_file} | yq .status) == "True" ]] +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: pg-tde + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + name: pg-tde + controller: true + blockOwnerDeletion: true +spec: + template: + metadata: + annotations: + pgv2.percona.com/tde-installed: "true" + spec: + containers: + - name: database + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /etc/database-containerinfo + name: database-containerinfo + readOnly: true + - mountPath: /pgconf/tde + name: pg-tde + readOnly: true + - mountPath: /etc/pgbackrest/conf.d + name: pgbackrest-config + readOnly: true + - mountPath: /etc/patroni + name: patroni-config + readOnly: true + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - mountPath: /dev/shm + name: dshm + - name: replication-cert-copy + - name: pgbackrest + - name: pgbackrest-config + volumes: + - name: cert-volume + - name: postgres-data + - name: database-containerinfo + - name: pg-tde + projected: + defaultMode: 384 + sources: + - secret: + items: + - key: token + path: token + name: vault-secret + - secret: + items: + - key: ca.crt + path: ca.crt + name: vault-secret + - name: pgbackrest-server + - name: pgbackrest-config + - name: patroni-config + - name: crunchy-bin + - name: tmp + - name: dshm +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: pg-tde +status: + state: ready +--- +apiVersion: upstream.pgv2.percona.com/v1beta1 +kind: PostgresCluster +metadata: + name: pg-tde +status: + pgTDERevision: 58dc94b68c +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: pg-tde + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/pg-tde/02-create-cluster.yaml b/e2e-tests/tests/pg-tde/02-create-cluster.yaml new file mode 100644 index 0000000000..c9a49ba290 --- /dev/null +++ b/e2e-tests/tests/pg-tde/02-create-cluster.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq '.spec.extensions.pg_tde.enabled = true' \ + | yq '.spec.extensions.pg_tde.vault.host = "https://vault-service.vault-service.svc:8200"' \ + | yq '.spec.extensions.pg_tde.vault.mountPath = "tde"' \ + | yq '.spec.extensions.pg_tde.vault.tokenSecret.name = "vault-secret"' \ + | yq '.spec.extensions.pg_tde.vault.tokenSecret.key = "token"' \ + | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret"' \ + | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde/03-write-data.yaml b/e2e-tests/tests/pg-tde/03-write-data.yaml new file mode 100644 index 0000000000..36ee115a06 --- /dev/null +++ b/e2e-tests/tests/pg-tde/03-write-data.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_psql_local \ + 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE myTable (id int PRIMARY KEY) USING tde_heap;' \ + "$(get_psql_uri pg-tde postgres)" + + run_psql_local \ + '\c myapp \\\ INSERT INTO myTable (id) VALUES (100500)' \ + "$(get_psql_uri pg-tde postgres)" diff --git a/e2e-tests/tests/pg-tde/04-assert.yaml b/e2e-tests/tests/pg-tde/04-assert.yaml new file mode 100644 index 0000000000..1aa25a7be8 --- /dev/null +++ b/e2e-tests/tests/pg-tde/04-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 05-verify-extension +data: + pg_tde_extension: ' pg_tde' +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: 05-verify-encryption +data: + pg_tde_is_encrypted: ' t' diff --git a/e2e-tests/tests/pg-tde/04-verify-encryption.yaml b/e2e-tests/tests/pg-tde/04-verify-encryption.yaml new file mode 100644 index 0000000000..b8bf2b47ca --- /dev/null +++ b/e2e-tests/tests/pg-tde/04-verify-encryption.yaml @@ -0,0 +1,23 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o xtrace + + source ../../functions + + result=$(run_psql_command \ + "SELECT extname FROM pg_extension WHERE extname = 'pg_tde';" \ + "$(get_psql_uri pg-tde postgres)") + kubectl -n "${NAMESPACE}" create configmap 05-verify-extension --from-literal=pg_tde_extension="$result" + + result=$(run_psql_command \ + "SELECT pg_tde_is_encrypted('myTable');" \ + "$(get_psql_uri pg-tde postgres)/myapp") + kubectl -n "${NAMESPACE}" create configmap 05-verify-encryption --from-literal=pg_tde_is_encrypted="$result" + + # pg_tde_verify_key will throw an error if it fails + run_psql_command \ + "SELECT pg_tde_verify_key();" \ + "$(get_psql_uri pg-tde postgres)/myapp" diff --git a/e2e-tests/tests/pg-tde/05-assert.yaml b/e2e-tests/tests/pg-tde/05-assert.yaml new file mode 100644 index 0000000000..f90d456b16 --- /dev/null +++ b/e2e-tests/tests/pg-tde/05-assert.yaml @@ -0,0 +1,31 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 660 +--- +kind: Job +apiVersion: batch/v1 +metadata: + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup1 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGBackup +metadata: + name: backup1 +spec: + pgCluster: pg-tde + repoName: repo1 + options: + - --type=full +status: + state: Succeeded diff --git a/e2e-tests/tests/pg-tde/05-create-backup.yaml b/e2e-tests/tests/pg-tde/05-create-backup.yaml new file mode 100644 index 0000000000..1789b2a9a1 --- /dev/null +++ b/e2e-tests/tests/pg-tde/05-create-backup.yaml @@ -0,0 +1,9 @@ +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGBackup +metadata: + name: backup1 +spec: + pgCluster: pg-tde + repoName: repo1 + options: + - --type=full diff --git a/e2e-tests/tests/pg-tde/06-write-data.yaml b/e2e-tests/tests/pg-tde/06-write-data.yaml new file mode 100644 index 0000000000..5d31274bf4 --- /dev/null +++ b/e2e-tests/tests/pg-tde/06-write-data.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_psql_local \ + '\c myapp \\\ INSERT INTO myTable (id) VALUES (100501)' \ + "$(get_psql_uri pg-tde postgres)" + + sleep 5 diff --git a/e2e-tests/tests/pg-tde/07-assert.yaml b/e2e-tests/tests/pg-tde/07-assert.yaml new file mode 100644 index 0000000000..4f6abe1437 --- /dev/null +++ b/e2e-tests/tests/pg-tde/07-assert.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 240 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: pg-tde +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore +metadata: + name: restore1 +spec: + pgCluster: pg-tde + repoName: repo1 +status: + state: Succeeded diff --git a/e2e-tests/tests/pg-tde/07-create-restore.yaml b/e2e-tests/tests/pg-tde/07-create-restore.yaml new file mode 100644 index 0000000000..8a666422a4 --- /dev/null +++ b/e2e-tests/tests/pg-tde/07-create-restore.yaml @@ -0,0 +1,7 @@ +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore +metadata: + name: restore1 +spec: + pgCluster: pg-tde + repoName: repo1 diff --git a/e2e-tests/tests/pg-tde/08-assert.yaml b/e2e-tests/tests/pg-tde/08-assert.yaml new file mode 100644 index 0000000000..4d8c494e75 --- /dev/null +++ b/e2e-tests/tests/pg-tde/08-assert.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 08-read-from-primary +data: + data: |2- + 100500 + 100501 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 08-read-from-replica-1 +data: + data: |2- + 100500 + 100501 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 08-read-from-replica-2 +data: + data: |2- + 100500 + 100501 \ No newline at end of file diff --git a/e2e-tests/tests/pg-tde/08-read-data.yaml b/e2e-tests/tests/pg-tde/08-read-data.yaml new file mode 100644 index 0000000000..eae4c8b930 --- /dev/null +++ b/e2e-tests/tests/pg-tde/08-read-data.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + primary=$(get_pod_by_role pg-tde primary name) + echo "Primary pod: ${primary}" + data=$(kubectl exec ${primary} -n "${NAMESPACE}" -- bash -c 'psql -q -t -d myapp -c "SELECT * from myTable;"') + kubectl create configmap -n "${NAMESPACE}" 08-read-from-primary --from-literal=data="${data}" + + t=1 + for i in $(kubectl get pods -n "${NAMESPACE}" -l postgres-operator.crunchydata.com/cluster=pg-tde,postgres-operator.crunchydata.com/role=replica -o jsonpath='{.items[*].metadata.name}'); do + echo "Replica pod: ${i}" + data=$(kubectl exec ${i} -n "${NAMESPACE}" -- bash -c 'psql -q -t -d myapp -c "SELECT * from myTable;"') + kubectl create configmap -n "${NAMESPACE}" 08-read-from-replica-${t} --from-literal=data="${data}" + t=$((t+1)) + done diff --git a/e2e-tests/tests/pg-tde/09-assert.yaml b/e2e-tests/tests/pg-tde/09-assert.yaml new file mode 100644 index 0000000000..b01b7a7610 --- /dev/null +++ b/e2e-tests/tests/pg-tde/09-assert.yaml @@ -0,0 +1,98 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: | + tmp=$(mktemp) + kubectl -n $NAMESPACE get pg pg-tde -o yaml | yq '.status.conditions.[] | select(.type == "PGTDEEnabled")' > ${tmp} + [[ $(cat ${tmp} | yq .message) == "pg_tde is enabled in PerconaPGCluster" ]] + [[ $(cat ${tmp} | yq .reason) == "Enabled" ]] + [[ $(cat ${tmp} | yq .status) == "True" ]] +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: pg-tde + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + name: pg-tde + controller: true + blockOwnerDeletion: true +spec: + template: + spec: + containers: + - name: database + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /etc/database-containerinfo + name: database-containerinfo + readOnly: true + - mountPath: /pgconf/tde + name: pg-tde + readOnly: true + - mountPath: /etc/pgbackrest/conf.d + name: pgbackrest-config + readOnly: true + - mountPath: /etc/patroni + name: patroni-config + readOnly: true + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - mountPath: /dev/shm + name: dshm + - name: replication-cert-copy + - name: pgbackrest + - name: pgbackrest-config + volumes: + - name: cert-volume + - name: postgres-data + - name: database-containerinfo + - name: pg-tde + projected: + defaultMode: 384 + sources: + - secret: + items: + - key: token + path: token + name: vault-secret-rotated + - secret: + items: + - key: ca.crt + path: ca.crt + name: vault-secret-rotated + - name: pgbackrest-server + - name: pgbackrest-config + - name: patroni-config + - name: crunchy-bin + - name: tmp + - name: dshm +status: + observedGeneration: 3 + replicas: 1 + readyReplicas: 1 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: pg-tde +status: + state: ready +--- +apiVersion: upstream.pgv2.percona.com/v1beta1 +kind: PostgresCluster +metadata: + name: pg-tde +status: + pgTDERevision: 6b5b555d8c diff --git a/e2e-tests/tests/pg-tde/09-change-vault-provider.yaml b/e2e-tests/tests/pg-tde/09-change-vault-provider.yaml new file mode 100644 index 0000000000..99d4d2bb5c --- /dev/null +++ b/e2e-tests/tests/pg-tde/09-change-vault-provider.yaml @@ -0,0 +1,32 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + vault_name=vault-service + + old_token=$(kubectl -n "${NAMESPACE}" get secret vault-secret -o jsonpath='{.data.token}' | base64 -d) + new_token=$(kubectl -n ${vault_name} exec ${vault_name}-0 -- \ + sh -c "VAULT_TOKEN=${old_token} vault token create -tls-skip-verify -format=json" | jq -r '.auth.client_token') + + kubectl -n "${NAMESPACE}" get secret vault-secret -o jsonpath='{.data.ca\.crt}' \ + | base64 -d > ${TEMP_DIR}/ca.crt + + kubectl -n "${NAMESPACE}" create secret generic vault-secret-rotated \ + --from-literal=token=${new_token} \ + --from-file=ca.crt=${TEMP_DIR}/ca.crt + + get_cr \ + | yq '.spec.extensions.pg_tde.enabled = true' \ + | yq '.spec.extensions.pg_tde.vault.host = "https://vault-service.vault-service.svc:8200"' \ + | yq '.spec.extensions.pg_tde.vault.mountPath = "tde"' \ + | yq '.spec.extensions.pg_tde.vault.tokenSecret.name = "vault-secret-rotated"' \ + | yq '.spec.extensions.pg_tde.vault.tokenSecret.key = "token"' \ + | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret-rotated"' \ + | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde/10-assert.yaml b/e2e-tests/tests/pg-tde/10-assert.yaml new file mode 100644 index 0000000000..01d608276f --- /dev/null +++ b/e2e-tests/tests/pg-tde/10-assert.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 10-read-after-change +data: + data: |2- + 100500 + 100501 diff --git a/e2e-tests/tests/pg-tde/10-verify-after-change.yaml b/e2e-tests/tests/pg-tde/10-verify-after-change.yaml new file mode 100644 index 0000000000..16bc38f779 --- /dev/null +++ b/e2e-tests/tests/pg-tde/10-verify-after-change.yaml @@ -0,0 +1,27 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + primary=$(get_pod_by_role pg-tde primary name) + data=$(kubectl exec ${primary} -n "${NAMESPACE}" -- bash -c 'psql -q -t -d myapp -c "SELECT * from myTable;"') + kubectl create configmap -n "${NAMESPACE}" 10-read-after-change --from-literal=data="${data}" + + run_psql_command \ + "SELECT pg_tde_verify_key();" \ + "$(get_psql_uri pg-tde postgres)/myapp" + + # Verify phase 2 cleanup: temp credential files should not exist on /pgdata + if kubectl exec ${primary} -n "${NAMESPACE}" -- test -f /pgdata/tde-new-token; then + echo "ERROR: /pgdata/tde-new-token should have been cleaned up after phase 2" + exit 1 + fi + if kubectl exec ${primary} -n "${NAMESPACE}" -- test -f /pgdata/tde-new-ca.crt; then + echo "ERROR: /pgdata/tde-new-ca.crt should have been cleaned up after phase 2" + exit 1 + fi diff --git a/e2e-tests/tests/pg-tde/11-assert.yaml b/e2e-tests/tests/pg-tde/11-assert.yaml new file mode 100644 index 0000000000..4a32219399 --- /dev/null +++ b/e2e-tests/tests/pg-tde/11-assert.yaml @@ -0,0 +1,91 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 180 +commands: + - script: | + tmp=$(mktemp) + kubectl -n $NAMESPACE get pg pg-tde -o yaml | yq '.status.conditions.[] | select(.type == "PGTDEEnabled")' > ${tmp} + [[ $(cat ${tmp} | yq .message) == "pg_tde is disabled in PerconaPGCluster" ]] + [[ $(cat ${tmp} | yq .reason) == "Disabled" ]] + [[ $(cat ${tmp} | yq .status) == "False" ]] +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: pg-tde + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + name: pg-tde + controller: true + blockOwnerDeletion: true +spec: + template: + spec: + containers: + - name: database + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /etc/database-containerinfo + name: database-containerinfo + readOnly: true + - mountPath: /pgconf/tde + name: pg-tde + readOnly: true + - mountPath: /etc/pgbackrest/conf.d + name: pgbackrest-config + readOnly: true + - mountPath: /etc/patroni + name: patroni-config + readOnly: true + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - mountPath: /dev/shm + name: dshm + - name: replication-cert-copy + - name: pgbackrest + - name: pgbackrest-config + volumes: + - name: cert-volume + - name: postgres-data + - name: database-containerinfo + - name: pg-tde + projected: + defaultMode: 384 + sources: + - secret: + items: + - key: token + path: token + name: vault-secret-rotated + - secret: + items: + - key: ca.crt + path: ca.crt + name: vault-secret-rotated + - name: pgbackrest-server + - name: pgbackrest-config + - name: patroni-config + - name: crunchy-bin + - name: tmp + - name: dshm +status: + observedGeneration: 4 + replicas: 1 + readyReplicas: 1 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: pg-tde +status: + state: ready diff --git a/e2e-tests/tests/pg-tde/11-disable-pgtde.yaml b/e2e-tests/tests/pg-tde/11-disable-pgtde.yaml new file mode 100644 index 0000000000..828b745f8d --- /dev/null +++ b/e2e-tests/tests/pg-tde/11-disable-pgtde.yaml @@ -0,0 +1,27 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o xtrace + + source ../../functions + + # pg_tde requires all encrypted objects to be dropped first + run_psql_command \ + "DROP TABLE mytable;" \ + "$(get_psql_uri pg-tde postgres)/myapp" + + run_psql_command \ + "CHECKPOINT;" \ + "$(get_psql_uri pg-tde postgres)/postgres" + + get_cr \ + | yq '.spec.extensions.pg_tde.enabled = false' \ + | yq '.spec.extensions.pg_tde.vault.host = "https://vault-service.vault-service.svc:8200"' \ + | yq '.spec.extensions.pg_tde.vault.mountPath = "tde"' \ + | yq '.spec.extensions.pg_tde.vault.tokenSecret.name = "vault-secret-rotated"' \ + | yq '.spec.extensions.pg_tde.vault.tokenSecret.key = "token"' \ + | yq '.spec.extensions.pg_tde.vault.caSecret.name = "vault-secret-rotated"' \ + | yq '.spec.extensions.pg_tde.vault.caSecret.key = "ca.crt"' \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/pg-tde/12-assert.yaml b/e2e-tests/tests/pg-tde/12-assert.yaml new file mode 100644 index 0000000000..1e461c5fe9 --- /dev/null +++ b/e2e-tests/tests/pg-tde/12-assert.yaml @@ -0,0 +1,67 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: pg-tde + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + name: pg-tde + controller: true + blockOwnerDeletion: true +spec: + template: + spec: + containers: + - name: database + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /etc/database-containerinfo + name: database-containerinfo + readOnly: true + - mountPath: /etc/pgbackrest/conf.d + name: pgbackrest-config + readOnly: true + - mountPath: /etc/patroni + name: patroni-config + readOnly: true + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - mountPath: /dev/shm + name: dshm + - name: replication-cert-copy + - name: pgbackrest + - name: pgbackrest-config + volumes: + - name: cert-volume + - name: postgres-data + - name: database-containerinfo + - name: pgbackrest-server + - name: pgbackrest-config + - name: patroni-config + - name: crunchy-bin + - name: tmp + - name: dshm +status: + observedGeneration: 5 + replicas: 1 + readyReplicas: 1 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: pg-tde +status: + state: ready diff --git a/e2e-tests/tests/pg-tde/12-remove-pgtde-config.yaml b/e2e-tests/tests/pg-tde/12-remove-pgtde-config.yaml new file mode 100644 index 0000000000..a0cb9a9c23 --- /dev/null +++ b/e2e-tests/tests/pg-tde/12-remove-pgtde-config.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o xtrace + + source ../../functions + + get_cr \ + | yq 'del(.spec.extensions.pg_tde)' \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/upgrade-minor/05-assert.yaml b/e2e-tests/tests/upgrade-minor/05-assert.yaml index 68dea800a5..961bcbabc5 100644 --- a/e2e-tests/tests/upgrade-minor/05-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/05-assert.yaml @@ -139,6 +139,9 @@ status: - type: APIGroupMigration reason: APIGroupMigrationNotNeeded status: "True" + - type: PGTDEEnabled + reason: Disabled + status: "False" pgbouncer: ready: 3 size: 3 diff --git a/e2e-tests/vars.sh b/e2e-tests/vars.sh index b3a1d6c44c..56d233d04a 100755 --- a/e2e-tests/vars.sh +++ b/e2e-tests/vars.sh @@ -61,6 +61,7 @@ export PGOV1_TAG=${PGOV1_TAG:-"1.4.0"} export PGOV1_VER=${PGOV1_VER:-"14"} export CPGO_VERSION=${CPGO_VERSION:-"5.8.7"} export MINIO_VER="5.4.0" +export VAULT_VER="0.32.0" # Add 'docker.io' for images that are provided without registry export REGISTRY_NAME="docker.io" diff --git a/internal/controller/pgupgrade/jobs.go b/internal/controller/pgupgrade/jobs.go index 51f2e0f92e..e8b4a64559 100644 --- a/internal/controller/pgupgrade/jobs.go +++ b/internal/controller/pgupgrade/jobs.go @@ -27,10 +27,20 @@ import ( // upgradeCommand returns an entrypoint that prepares the filesystem for // and performs a PostgreSQL major version upgrade using pg_upgrade. -func upgradeCommand(oldVersion, newVersion int, fetchKeyCommand string, availableCPUs int) []string { +func upgradeCommand(oldVersion, newVersion int, fetchKeyCommand string, availableCPUs int, pgTDE bool) []string { // Use multiple CPUs when three or more are available. argJobs := fmt.Sprintf(` --jobs=%d`, max(1, availableCPUs-1)) + // K8SPG-911: Running pg_upgrade on a cluster with pg_tde corrupts the + // encryption metadata. pg_tde_upgrade wraps pg_upgrade with the same + // arguments: it copies the pg_tde directory from the old data directory + // and uses pg_tde_resetwal instead of pg_resetwal. + // - https://docs.percona.com/pg-tde/command-line-tools/pg-tde-upgrade.html + upgradeBinary := `pg_upgrade` + if pgTDE { + upgradeBinary = `pg_tde_upgrade` + } + // if the fetch key command is set for TDE, provide the value during initialization initdb := `/usr/pgsql-"${new_version}"/bin/initdb -k -D /pgdata/pg"${new_version}"` if fetchKeyCommand != "" { @@ -88,15 +98,15 @@ func upgradeCommand(oldVersion, newVersion int, fetchKeyCommand string, availabl // Before the actual upgrade is run, we will run the upgrade --check to // verify everything before actually changing any data. - `echo -e "Step 5: Running pg_upgrade check...\n"`, - `time /usr/pgsql-"${new_version}"/bin/pg_upgrade --old-bindir /usr/pgsql-"${old_version}"/bin \`, + `echo -e "Step 5: Running ` + upgradeBinary + ` check...\n"`, + `time /usr/pgsql-"${new_version}"/bin/` + upgradeBinary + ` --old-bindir /usr/pgsql-"${old_version}"/bin \`, `--new-bindir /usr/pgsql-"${new_version}"/bin --old-datadir /pgdata/pg"${old_version}"\`, ` --new-datadir /pgdata/pg"${new_version}" --link --check` + argJobs, // Assuming the check completes successfully, the pg_upgrade command will // be run that actually prepares the upgraded pgdata directory. - `echo -e "\nStep 6: Running pg_upgrade...\n"`, - `time /usr/pgsql-"${new_version}"/bin/pg_upgrade --old-bindir /usr/pgsql-"${old_version}"/bin \`, + `echo -e "\nStep 6: Running ` + upgradeBinary + `...\n"`, + `time /usr/pgsql-"${new_version}"/bin/` + upgradeBinary + ` --old-bindir /usr/pgsql-"${old_version}"/bin \`, `--new-bindir /usr/pgsql-"${new_version}"/bin --old-datadir /pgdata/pg"${old_version}" \`, `--new-datadir /pgdata/pg"${new_version}" --link` + argJobs, @@ -127,7 +137,7 @@ func largestWholeCPU(resources corev1.ResourceRequirements) int { // directory of the startup instance. func (r *PGUpgradeReconciler) generateUpgradeJob( ctx context.Context, upgrade *v1beta1.PGUpgrade, - startup *appsv1.StatefulSet, fetchKeyCommand string, + startup *appsv1.StatefulSet, fetchKeyCommand string, pgTDE bool, ) *batchv1.Job { job := &batchv1.Job{} job.SetGroupVersionKind(batchv1.SchemeGroupVersion.WithKind("Job")) @@ -228,7 +238,8 @@ func (r *PGUpgradeReconciler) generateUpgradeJob( upgrade.Spec.FromPostgresVersion, upgrade.Spec.ToPostgresVersion, fetchKeyCommand, - wholeCPUs), + wholeCPUs, + pgTDE), Image: pgUpgradeContainerImage(upgrade), ImagePullPolicy: upgrade.Spec.ImagePullPolicy, Resources: upgrade.Spec.Resources, diff --git a/internal/controller/pgupgrade/jobs_test.go b/internal/controller/pgupgrade/jobs_test.go index 1f1fef12dd..d50b1c65d8 100644 --- a/internal/controller/pgupgrade/jobs_test.go +++ b/internal/controller/pgupgrade/jobs_test.go @@ -83,7 +83,7 @@ func TestUpgradeCommand(t *testing.T) { {CPUs: 3, Jobs: "--jobs=2"}, {CPUs: 10, Jobs: "--jobs=9"}, } { - command := upgradeCommand(10, 11, "", tt.CPUs) + command := upgradeCommand(10, 11, "", tt.CPUs, false) assert.Assert(t, len(command) > 3) assert.DeepEqual(t, []string{"bash", "-ceu", "--"}, command[:3]) @@ -93,6 +93,17 @@ func TestUpgradeCommand(t *testing.T) { expectScript(t, script) } }) + + // K8SPG-911: pg_tde_upgrade replaces pg_upgrade when pg_tde is enabled. + t.Run("PGTDE", func(t *testing.T) { + script := upgradeCommand(17, 18, "", 0, true)[3] + assert.Assert(t, cmp.Contains(script, + `/usr/pgsql-"${new_version}"/bin/pg_tde_upgrade --old-bindir`)) + assert.Assert(t, !strings.Contains(script, `bin/pg_upgrade `), + "expected no pg_upgrade invocation, got:\n%s", script) + + expectScript(t, script) + }) } func TestGenerateUpgradeJob(t *testing.T) { @@ -141,7 +152,7 @@ func TestGenerateUpgradeJob(t *testing.T) { }, } - job := reconciler.generateUpgradeJob(ctx, upgrade, startup, "") + job := reconciler.generateUpgradeJob(ctx, upgrade, startup, "", false) assert.Assert(t, cmp.MarshalMatches(job, ` apiVersion: batch/v1 kind: Job @@ -250,7 +261,7 @@ status: {} longNameUpgrade.Spec.FromPostgresVersion = 14 longNameUpgrade.Spec.ToPostgresVersion = 15 - longJob := reconciler.generateUpgradeJob(ctx, longNameUpgrade, startup, "") + longJob := reconciler.generateUpgradeJob(ctx, longNameUpgrade, startup, "", false) // Verify the job name fits within DNS limits and has the correct format assert.Assert(t, len(longJob.Name) <= 63, "job name %q exceeds 63 characters", longJob.Name) @@ -260,7 +271,7 @@ status: {} assert.Assert(t, regexp.MustCompile(`-[a-zA-Z0-9]{4}$`).MatchString(longJob.Name), "job name %q should end with -<4 alphanumeric chars>", longJob.Name) // Verify the suffix is deterministic (same input always produces same output) - longJob2 := reconciler.generateUpgradeJob(ctx, longNameUpgrade, startup, "") + longJob2 := reconciler.generateUpgradeJob(ctx, longNameUpgrade, startup, "", false) assert.Assert(t, longJob.Name == longJob2.Name, "job name should be deterministic: %q vs %q", longJob.Name, longJob2.Name) }) @@ -271,11 +282,11 @@ status: {} })) ctx := feature.NewContext(context.Background(), gate) - job := reconciler.generateUpgradeJob(ctx, upgrade, startup, "") + job := reconciler.generateUpgradeJob(ctx, upgrade, startup, "", false) assert.Assert(t, cmp.MarshalContains(job, `--jobs=2`)) }) - tdeJob := reconciler.generateUpgradeJob(ctx, upgrade, startup, "echo testKey") + tdeJob := reconciler.generateUpgradeJob(ctx, upgrade, startup, "echo testKey", false) assert.Assert(t, cmp.MarshalContains(tdeJob, `/usr/pgsql-"${new_version}"/bin/initdb -k -D /pgdata/pg"${new_version}" --encryption-key-command "echo testKey"`)) } diff --git a/internal/controller/pgupgrade/pgupgrade_controller.go b/internal/controller/pgupgrade/pgupgrade_controller.go index 1e37605487..19eab0b097 100644 --- a/internal/controller/pgupgrade/pgupgrade_controller.go +++ b/internal/controller/pgupgrade/pgupgrade_controller.go @@ -466,7 +466,9 @@ func (r *PGUpgradeReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // TODO: error from apply could mean that the job exists with a different spec. if err == nil && !upgradeJobComplete { err = errors.WithStack(r.apply(ctx, - r.generateUpgradeJob(ctx, upgrade, world.ClusterPrimary, config.FetchKeyCommand(&world.Cluster.Spec)))) + r.generateUpgradeJob(ctx, upgrade, world.ClusterPrimary, + config.FetchKeyCommand(&world.Cluster.Spec), + world.Cluster.Spec.Extensions.PGTDE.Enabled))) // K8SPG-911 } // Create the jobs to remove the data from the replicas, as long as diff --git a/internal/controller/postgrescluster/cluster.go b/internal/controller/postgrescluster/cluster.go index a8e4f56e2c..1bba315bf3 100644 --- a/internal/controller/postgrescluster/cluster.go +++ b/internal/controller/postgrescluster/cluster.go @@ -19,6 +19,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/patroni" "github.com/percona/percona-postgresql-operator/v2/internal/pki" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + "github.com/percona/percona-postgresql-operator/v2/internal/util" "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" ) @@ -288,7 +289,7 @@ func (r *Reconciler) reconcileDataSource(ctx context.Context, ) (bool, error) { // a hash func to hash the pgBackRest restore options hashFunc := func(jobConfigs []string) (string, error) { - return safeHash32(func(w io.Writer) (err error) { + return util.SafeHash32(func(w io.Writer) (err error) { for _, o := range jobConfigs { _, err = w.Write([]byte(o)) } diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 35d6731aeb..16c9f409e7 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -50,6 +50,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/pgmonitor" "github.com/percona/percona-postgresql-operator/v2/internal/pgstatmonitor" "github.com/percona/percona-postgresql-operator/v2/internal/pgstatstatements" + "github.com/percona/percona-postgresql-operator/v2/internal/pgtde" "github.com/percona/percona-postgresql-operator/v2/internal/pki" "github.com/percona/percona-postgresql-operator/v2/internal/pmm" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" @@ -290,6 +291,15 @@ func (r *Reconciler) Reconcile( if cluster.Spec.Extensions.SetUser { setuser.PostgreSQLParameters(&pgParameters) } + + pgTDECondition := meta.FindStatusCondition(cluster.Status.Conditions, + v1beta1.PGTDEEnabled) + pgTDEEnabled := pgTDECondition != nil && pgTDECondition.Status == metav1.ConditionTrue + // pg_tde should be removed from shared libraries only after extension is dropped + if cluster.Spec.Extensions.PGTDE.Enabled || pgTDEEnabled { + pgtde.PostgreSQLParameters(&pgParameters) + } + pgbackrest.PostgreSQL(cluster, &pgParameters, backupsSpecFound) pgmonitor.PostgreSQLParameters(cluster, &pgParameters) @@ -429,7 +439,10 @@ func (r *Reconciler) Reconcile( } if err == nil { - err = r.reconcilePostgresDatabases(ctx, cluster, instances) + err = r.reconcilePostgresDatabases(ctx, cluster, instances, patchClusterStatus) + } + if err == nil { + err = r.reconcilePGTDEProviders(ctx, cluster, instances, patchClusterStatus) } if err == nil { err = r.reconcilePostgresUsers(ctx, cluster, instances) diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go index 3e849c3f39..8446e30128 100644 --- a/internal/controller/postgrescluster/instance.go +++ b/internal/controller/postgrescluster/instance.go @@ -19,6 +19,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -35,6 +36,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/naming" "github.com/percona/percona-postgresql-operator/v2/internal/patroni" "github.com/percona/percona-postgresql-operator/v2/internal/pgbackrest" + "github.com/percona/percona-postgresql-operator/v2/internal/pgtde" "github.com/percona/percona-postgresql-operator/v2/internal/pki" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" "github.com/percona/percona-postgresql-operator/v2/percona/certmanager" @@ -290,6 +292,31 @@ func (observed *observedInstances) writablePod(container string) (*corev1.Pod, * return nil, nil } +// runningPods returns the Pod of every non-terminating instance whose named +// container is running, and whether that accounts for every instance in the +// cluster. Callers that must reach the whole cluster, rather than any one +// member of it, should wait while complete is false. +func (observed *observedInstances) runningPods(container string) (pods []*corev1.Pod, complete bool) { + if observed == nil { + return nil, false + } + + complete = true + for _, instance := range observed.forCluster { + if terminating, known := instance.IsTerminating(); terminating || !known { + complete = false + continue + } + if running, known := instance.IsRunning(container); !running || !known || len(instance.Pods) == 0 { + complete = false + continue + } + pods = append(pods, instance.Pods[0]) + } + + return pods, complete +} + // +kubebuilder:rbac:groups="",resources="pods",verbs={list} // +kubebuilder:rbac:groups="apps",resources="statefulsets",verbs={list} @@ -1212,6 +1239,25 @@ func (r *Reconciler) reconcileInstance( postgresDataVolume, postgresWALVolume, tablespaceVolumes, &instance.Spec.Template.Spec) + // K8SPG-911: reconcilePGTDEProviders has not repointed the key provider + // at the credentials in the spec yet, so keep mounting the ones it does + // name. Rolling now would leave pg_tde reading a token the provider was + // never told about. Once phase 1 lands, the phase moves on and the + // volume updates, which is what restarts the Pods for phase 2. + if cluster.Spec.Extensions.PGTDE.Vault != nil { + change, changeErr := pgtde.VaultChangeFor(cluster) + if changeErr != nil { + // Without the revisions there is no way to tell a pending + // change from a settled one. Hold the volume: a Pod that keeps + // its credentials can be rolled later, one that loses them + // cannot get them back. + log.Error(changeErr, "keeping the pg_tde vault volume") + } + if changeErr != nil || change.Phase == pgtde.StageCredentials { + pgtde.PreserveOldTDEVolume(&instance.Spec.Template.Spec, existing) + } + } + if backupsSpecFound { addPGBackRestToInstancePodSpec( ctx, cluster, instanceCertificates, &instance.Spec.Template.Spec) @@ -1333,6 +1379,18 @@ func generateInstanceStatefulSetIntent(_ context.Context, }, ) } + + pgTDECondition := meta.FindStatusCondition(cluster.Status.Conditions, + v1beta1.PGTDEEnabled) + pgTDEEnabled := pgTDECondition != nil && pgTDECondition.Status == metav1.ConditionTrue + // we should restart pods only after extension is dropped + if cluster.Spec.Extensions.PGTDE.Enabled || pgTDEEnabled { + sts.Spec.Template.Annotations = naming.Merge( + sts.Spec.Template.Annotations, + map[string]string{naming.TDEInstalledAnnotation: "true"}, + ) + } + sts.Spec.Template.Labels = naming.Merge( cluster.Spec.Metadata.GetLabelsOrNil(), spec.Metadata.GetLabelsOrNil(), diff --git a/internal/controller/postgrescluster/pgadmin.go b/internal/controller/postgrescluster/pgadmin.go index 4f1b527cd5..9ab6ba9029 100644 --- a/internal/controller/postgrescluster/pgadmin.go +++ b/internal/controller/postgrescluster/pgadmin.go @@ -24,6 +24,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/naming" "github.com/percona/percona-postgresql-operator/v2/internal/pgadmin" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + "github.com/percona/percona-postgresql-operator/v2/internal/util" "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" ) @@ -467,7 +468,7 @@ func (r *Reconciler) reconcilePGAdminUsers( return pgadmin.WriteUsersInPGAdmin(ctx, cluster, exec, specUsers, passwords) } - revision, err := safeHash32(func(hasher io.Writer) error { + revision, err := util.SafeHash32(func(hasher io.Writer) error { // Discard log messages about executing. return write(logging.NewContext(ctx, logging.Discard()), func( _ context.Context, stdin io.Reader, _, _ io.Writer, command ...string, diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index 09c9a5fc1c..e37ff04d3c 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -1338,7 +1338,7 @@ func (r *Reconciler) reconcileRestoreJob(ctx context.Context, // NOTE (andrewlecuyer): Forcing users to put each argument separately might prevent the need // to do any escaping or use eval. cmd := pgbackrest.RestoreCommand(pgdata, hugePagesSetting, config.FetchKeyCommand(&cluster.Spec), - pgtablespaceVolumes, strings.Join(opts, " ")) + pgtablespaceVolumes, cluster.Spec.Extensions.PGTDE.Enabled, strings.Join(opts, " ")) // create the volume resources required for the postgres data directory dataVolumeMount := postgres.DataVolumeMount() @@ -1382,6 +1382,11 @@ func (r *Reconciler) reconcileRestoreJob(ctx context.Context, volumeMounts = append(volumeMounts, tablespaceVolumeMount) } + if vault := cluster.Spec.Extensions.PGTDE.Vault; vault != nil { + volumeMounts = append(volumeMounts, postgres.PGTDEVolumeMount()) + volumes = append(volumes, postgres.PGTDEVolume(vault)) + } + restoreJob := &batchv1.Job{} if err := r.generateRestoreJobIntent(cluster, configHash, instanceName, cmd, volumeMounts, volumes, dataSource, restoreJob); err != nil { diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index e3fdcbc9be..a7097581ae 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -25,6 +25,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/pgbouncer" "github.com/percona/percona-postgresql-operator/v2/internal/pki" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + "github.com/percona/percona-postgresql-operator/v2/internal/util" "github.com/percona/percona-postgresql-operator/v2/percona/certmanager" "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" ) @@ -153,7 +154,7 @@ func (r *Reconciler) reconcilePGBouncerInPostgreSQL( // First, calculate a hash of the SQL that should be executed in PostgreSQL. - revision, err := safeHash32(func(hasher io.Writer) error { + revision, err := util.SafeHash32(func(hasher io.Writer) error { // Discard log messages from the pgbouncer package about executing SQL. // Nothing is being "executed" yet. return action(logging.NewContext(ctx, logging.Discard()), func( diff --git a/internal/controller/postgrescluster/pgmonitor.go b/internal/controller/postgrescluster/pgmonitor.go index 158ff0f31d..5b08c1d08d 100644 --- a/internal/controller/postgrescluster/pgmonitor.go +++ b/internal/controller/postgrescluster/pgmonitor.go @@ -108,7 +108,7 @@ func (r *Reconciler) reconcilePGMonitorExporter(ctx context.Context, } } - revision, err := safeHash32(func(hasher io.Writer) error { + revision, err := util.SafeHash32(func(hasher io.Writer) error { // Discard log message from pgmonitor package about executing SQL. // Nothing is being "executed" yet. return action(logging.NewContext(ctx, logging.Discard()), func( diff --git a/internal/controller/postgrescluster/pgtde_test.go b/internal/controller/postgrescluster/pgtde_test.go new file mode 100644 index 0000000000..8c0d957fe9 --- /dev/null +++ b/internal/controller/postgrescluster/pgtde_test.go @@ -0,0 +1,1571 @@ +// Copyright 2021 - 2024 Crunchy Data Solutions, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +package postgrescluster + +import ( + "context" + "io" + "strconv" + "strings" + "testing" + + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "gotest.tools/v3/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/percona/percona-postgresql-operator/v2/internal/controller/runtime" + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/pgtde" + "github.com/percona/percona-postgresql-operator/v2/internal/pki" + "github.com/percona/percona-postgresql-operator/v2/internal/testing/events" + "github.com/percona/percona-postgresql-operator/v2/internal/testing/require" + "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +// execCall records a single invocation of Reconciler.PodExec. +type execCall struct { + namespace string + pod string + container string + stdin string + command []string +} + +// execRecorder returns a PodExec function that appends every call to calls and +// returns the error produced by result, if any. +func execRecorder(calls *[]execCall, result func(call execCall) error) func( + ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, +) error { + return func( + ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + call := execCall{ + namespace: namespace, + pod: pod, + container: container, + command: command, + } + if stdin != nil { + b, err := io.ReadAll(stdin) + if err != nil { + return err + } + call.stdin = string(b) + } + + *calls = append(*calls, call) + + if result != nil { + if err := result(call); err != nil { + return err + } + } + + // Stand in for the "wc -c" that fetchSecretToTempFile appends to its + // write command to detect short writes. + if len(command) > 2 && strings.Contains(command[2], "wc -c") { + _, _ = io.WriteString(stdout, strconv.Itoa(len(call.stdin))+"\n") + } + return nil + } +} + +// tdeVaultSpec is the vault configuration shared by the tests below. +func tdeVaultSpec() *v1beta1.PGTDEVaultSpec { + return &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example:8200", + MountPath: "tde", + TokenSecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-secret", Key: "token", + }, + CASecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-secret", Key: "ca.crt", + }, + } +} + +// tdeInstance builds an observed instance that is running, writable and whose +// Pod matches its PodTemplate, i.e. one that reconcilePGTDEProviders accepts. +func tdeInstance(annotations map[string]string) *Instance { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "pgc1-instance1-abcd-0", + Labels: map[string]string{ + appsv1.StatefulSetRevisionLabel: "rev-1", + }, + Annotations: map[string]string{ + "status": `{"role":"primary"}`, + }, + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{{ + Name: naming.ContainerDatabase, + State: corev1.ContainerState{Running: new(corev1.ContainerStateRunning)}, + }}, + }, + } + for k, v := range annotations { + pod.Annotations[k] = v + } + + return &Instance{ + Name: "instance1-abcd", + Pods: []*corev1.Pod{pod}, + Runner: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + Status: appsv1.StatefulSetStatus{ + ObservedGeneration: 2, + UpdateRevision: "rev-1", + }, + }, + } +} + +func TestPGTDEVaultRevision(t *testing.T) { + t.Parallel() + + vault := tdeVaultSpec() + tokenPath, caPath := pgtde.VaultCredentialPaths(vault) + + base, err := pgtde.VaultRevision(vault, tokenPath, caPath) + assert.NilError(t, err) + assert.Assert(t, base != "") + + t.Run("Deterministic", func(t *testing.T) { + again, err := pgtde.VaultRevision(tdeVaultSpec(), tokenPath, caPath) + assert.NilError(t, err) + assert.Equal(t, base, again, "same input should hash the same") + }) + + t.Run("TempPathsDiffer", func(t *testing.T) { + tempToken, tempCA := pgtde.TempVaultCredentialPaths(vault) + temp, err := pgtde.VaultRevision(vault, tempToken, tempCA) + assert.NilError(t, err) + assert.Assert(t, temp != base, + "temp revision must differ from standard revision; the two-phase "+ + "provider change relies on telling them apart") + }) + + // Every field that influences how PostgreSQL reaches Vault must change the + // revision, otherwise a configuration change is silently never applied. + for _, tc := range []struct { + name string + mutate func(*v1beta1.PGTDEVaultSpec) + }{ + {"Host", func(v *v1beta1.PGTDEVaultSpec) { v.Host = "https://other:8200" }}, + {"MountPath", func(v *v1beta1.PGTDEVaultSpec) { v.MountPath = "other" }}, + {"TokenSecretName", func(v *v1beta1.PGTDEVaultSpec) { v.TokenSecret.Name = "other" }}, + {"TokenSecretKey", func(v *v1beta1.PGTDEVaultSpec) { v.TokenSecret.Key = "other" }}, + {"CASecretName", func(v *v1beta1.PGTDEVaultSpec) { v.CASecret.Name = "other" }}, + {"CASecretKey", func(v *v1beta1.PGTDEVaultSpec) { v.CASecret.Key = "other" }}, + } { + t.Run(tc.name, func(t *testing.T) { + changed := tdeVaultSpec() + tc.mutate(changed) + + // Recompute the paths; some of the fields above feed into them. + token, ca := pgtde.VaultCredentialPaths(changed) + rev, err := pgtde.VaultRevision(changed, token, ca) + assert.NilError(t, err) + assert.Assert(t, rev != base, "changing %s should change the revision", tc.name) + }) + } + + // Host and MountPath are hashed one after another and neither feeds into + // the credential paths, so they isolate how the fields are delimited from + // everything else that goes into the revision. + hostMount := func(host, mountPath string) string { + t.Helper() + vault := tdeVaultSpec() + vault.Host, vault.MountPath = host, mountPath + + token, ca := pgtde.VaultCredentialPaths(vault) + revision, err := pgtde.VaultRevision(vault, token, ca) + assert.NilError(t, err) + return revision + } + + // Without a delimiter, moving a character across a field boundary produces + // the same revision, and reconcilePGTDEProviders takes its "matches the + // spec" early return on a Vault the cluster has never been pointed at. + t.Run("FieldBoundaries", func(t *testing.T) { + assert.Assert(t, + hostMount("https://vault:8200", "secret/data") != + hostMount("https://vault:8200secret", "/data"), + "a character moved from one field to the next must change the revision") + }) + + // Delimiting with a quote is only injective when a quote appearing inside a + // value is escaped. + t.Run("QuotesInValues", func(t *testing.T) { + assert.Assert(t, hostMount(`a"`, `b`) != hostMount(`a`, `"b`), + "a quote inside a value must not fake a field boundary") + }) +} + +func TestPreserveOldTDEVolume(t *testing.T) { + t.Parallel() + + oldVolume := corev1.Volume{ + Name: naming.PGTDEVolume, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "old-secret"}, + }, + }}, + }, + }, + } + newVolume := corev1.Volume{ + Name: naming.PGTDEVolume, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "new-secret"}, + }, + }}, + }, + }, + } + + runnerWith := func(volumes ...corev1.Volume) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Volumes: volumes}, + }, + }, + } + } + + t.Run("Replaces", func(t *testing.T) { + podSpec := &corev1.PodSpec{Volumes: []corev1.Volume{ + {Name: "pgdata"}, newVolume, {Name: "tmp"}, + }} + + pgtde.PreserveOldTDEVolume(podSpec, runnerWith(oldVolume, corev1.Volume{Name: "pgdata"})) + + assert.Equal(t, len(podSpec.Volumes), 3, "no volumes should be added or removed") + assert.Equal(t, podSpec.Volumes[0].Name, "pgdata", "unrelated volumes keep their order") + assert.Equal(t, podSpec.Volumes[2].Name, "tmp") + assert.Equal(t, + podSpec.Volumes[1].Projected.Sources[0].Secret.Name, "old-secret", + "the running Pod's TDE volume should be kept") + }) + + t.Run("NoVolumeInRunner", func(t *testing.T) { + podSpec := &corev1.PodSpec{Volumes: []corev1.Volume{newVolume}} + + pgtde.PreserveOldTDEVolume(podSpec, runnerWith(corev1.Volume{Name: "pgdata"})) + + assert.Equal(t, + podSpec.Volumes[0].Projected.Sources[0].Secret.Name, "new-secret", + "without an old volume to preserve the intent is left alone") + }) + + t.Run("NoVolumeInPodSpec", func(t *testing.T) { + podSpec := &corev1.PodSpec{Volumes: []corev1.Volume{{Name: "pgdata"}}} + + pgtde.PreserveOldTDEVolume(podSpec, runnerWith(oldVolume)) + + assert.Equal(t, len(podSpec.Volumes), 1, + "the old volume should not be grafted onto a Pod that has none") + assert.Equal(t, podSpec.Volumes[0].Name, "pgdata") + }) +} + +// TestScaleUpInstancesPreservesTDEVolume covers the wiring around +// preserveOldTDEVolume rather than the helper itself. scaleUpInstances passes +// the observed StatefulSet as the intent to build into, and reconcileInstance +// empties that object before generating the new Pod spec. Reading the old +// volume from the observed instance therefore copies the new volume onto +// itself, and every unit test of the helper still passes because it builds the +// two objects separately. +func TestScaleUpInstancesPreservesTDEVolume(t *testing.T) { + ctx := context.Background() + _, cc := setupKubernetes(t) + require.ParallelCapacity(t, 1) + + ns := setupNamespace(t, cc) + + r := &Reconciler{ + Client: cc, + Owner: client.FieldOwner(t.Name()), + Recorder: new(record.FakeRecorder), + Tracer: otel.Tracer(t.Name()), + } + + rootCA, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) + + vaultWith := func(secretName string) *v1beta1.PGTDEVaultSpec { + return &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + MountPath: "secret/data", + TokenSecret: v1beta1.PGTDESecretObjectReference{Name: secretName, Key: "token"}, + } + } + + cluster := testCluster() + cluster.Namespace = ns.Name + cluster.Spec.PostgresVersion = 17 + cluster.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + Vault: vaultWith("vault-old"), + } + assert.NilError(t, cluster.Default(ctx, nil)) + assert.NilError(t, cc.Create(ctx, cluster)) + + set := &cluster.Spec.InstanceSets[0] + + // observe lists the StatefulSets the previous round applied, the way + // Reconcile does at the top of each pass. + observe := func() *observedInstances { + t.Helper() + var runners appsv1.StatefulSetList + assert.NilError(t, cc.List(ctx, &runners, client.InNamespace(ns.Name))) + return newObservedInstances(cluster, runners.Items, nil) + } + + scaleUp := func(observed *observedInstances) { + t.Helper() + object := func(name string) metav1.ObjectMeta { + return metav1.ObjectMeta{Namespace: ns.Name, Name: name} + } + _, err := r.scaleUpInstances(ctx, cluster, observed, set, + &corev1.ConfigMap{ObjectMeta: object("cluster-config")}, + &corev1.Secret{ObjectMeta: object("cluster-replication")}, + rootCA, + &corev1.Service{ObjectMeta: object("cluster-pods")}, + &corev1.ServiceAccount{ObjectMeta: object("cluster-instance")}, + &corev1.Service{ObjectMeta: object("cluster-ha")}, + clusterCertSecretProjection(&corev1.Secret{ObjectMeta: object("cluster-cert")}), + nil, 1, nil, nil, nil, false) + assert.NilError(t, err) + } + + // appliedTokenSecret returns the Secret projected into the pg-tde volume of + // the instance StatefulSet as it exists in the API. + appliedTokenSecret := func() string { + t.Helper() + var runners appsv1.StatefulSetList + assert.NilError(t, cc.List(ctx, &runners, client.InNamespace(ns.Name))) + assert.Equal(t, len(runners.Items), 1) + + for _, volume := range runners.Items[0].Spec.Template.Spec.Volumes { + if volume.Name == naming.PGTDEVolume { + return volume.Projected.Sources[0].Secret.Name + } + } + t.Fatalf("no %q volume in %v", naming.PGTDEVolume, + runners.Items[0].Spec.Template.Spec.Volumes) + return "" + } + + // revisionFor is the value reconcilePGTDEProviders stores in + // Status.PGTDERevision once the provider names the given paths. + revisionFor := func(vault *v1beta1.PGTDEVaultSpec, temp bool) string { + t.Helper() + paths := pgtde.VaultCredentialPaths + if temp { + paths = pgtde.TempVaultCredentialPaths + } + tokenPath, caPath := paths(vault) + revision, err := pgtde.VaultRevision(vault, tokenPath, caPath) + assert.NilError(t, err) + return revision + } + + // Initial setup: no provider has been configured, so there is nothing to + // hold and the Pod gets the Secret named in the spec. + scaleUp(observe()) + assert.Equal(t, appliedTokenSecret(), "vault-old") + + cluster.Status.PGTDERevision = revisionFor(vaultWith("vault-old"), false) + + // The user points pg_tde at a different Vault token. Until phase 1 has run + // the provider still names the old credentials, so the Pod has to keep + // mounting them; rolling now would leave pg_tde unable to fetch its key. + cluster.Spec.Extensions.PGTDE.Vault = vaultWith("vault-new") + scaleUp(observe()) + assert.Equal(t, appliedTokenSecret(), "vault-old", + "the vault volume should be held until the provider change has run") + + // Phase 1 has run: the provider now names the staged credentials on the + // data volume, so the hold is released and the Pods roll. + cluster.Status.PGTDERevision = revisionFor(vaultWith("vault-new"), true) + scaleUp(observe()) + assert.Equal(t, appliedTokenSecret(), "vault-new", + "the hold should be released once the provider names the staged credentials") +} + +func TestStageVaultCredentials(t *testing.T) { + t.Parallel() + ctx := context.Background() + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "vault-secret"}, + Data: map[string][]byte{ + "token": []byte("hvs.sometoken"), + "ca.crt": []byte("-----BEGIN CERTIFICATE-----"), + }, + } + + newPods := func(names ...string) []*corev1.Pod { + pods := make([]*corev1.Pod, 0, len(names)) + for _, name := range names { + pods = append(pods, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: name}, + }) + } + return pods + } + + vault := tdeVaultSpec() + tokenPath, caPath := pgtde.TempVaultCredentialPaths(vault) + + t.Run("WritesToEveryPod", func(t *testing.T) { + var calls []execCall + k8s := fake.NewClientBuilder().WithObjects(secret).Build() + pods := newPods("pgc1-instance1-abcd-0", "pgc1-instance2-efgh-0", "pgc1-instance3-ijkl-0") + + assert.NilError(t, stagePGTDEVaultCredentials(ctx, k8s, execRecorder(&calls, nil), + "ns1", vault, pods, naming.ContainerDatabase, tokenPath, caPath)) + + // pg_tde names one path cluster-wide, but each instance has its own + // /pgdata, so every one of them needs its own copy. + assert.Equal(t, len(calls), 6, "two files on each of three instances") + + for i, pod := range pods { + token, ca := calls[i*2], calls[i*2+1] + + assert.Equal(t, token.pod, pod.Name) + assert.Equal(t, token.stdin, "hvs.sometoken") + assert.Assert(t, strings.Contains(token.command[2], tokenPath)) + + assert.Equal(t, ca.pod, pod.Name) + assert.Equal(t, ca.stdin, "-----BEGIN CERTIFICATE-----") + assert.Assert(t, strings.Contains(ca.command[2], caPath)) + } + }) + + t.Run("ReadsEachSecretOnce", func(t *testing.T) { + var calls []execCall + gets := 0 + k8s := &countingReader{ + Reader: fake.NewClientBuilder().WithObjects(secret).Build(), + gets: &gets, + } + + assert.NilError(t, stagePGTDEVaultCredentials(ctx, k8s, execRecorder(&calls, nil), + "ns1", vault, newPods("a", "b", "c", "d"), naming.ContainerDatabase, + tokenPath, caPath)) + + assert.Equal(t, gets, 2, + "the token and CA Secrets should be read once, not once per instance") + }) + + t.Run("WithoutCASecret", func(t *testing.T) { + var calls []execCall + k8s := fake.NewClientBuilder().WithObjects(secret).Build() + + noCA := tdeVaultSpec() + noCA.CASecret = v1beta1.PGTDESecretObjectReference{} + _, noCAPath := pgtde.TempVaultCredentialPaths(noCA) + + assert.NilError(t, stagePGTDEVaultCredentials(ctx, k8s, execRecorder(&calls, nil), + "ns1", noCA, newPods("a", "b"), naming.ContainerDatabase, + tokenPath, noCAPath)) + + assert.Equal(t, len(calls), 2, "only the token is staged") + }) + + t.Run("MissingSecretWritesNothing", func(t *testing.T) { + var calls []execCall + k8s := fake.NewClientBuilder().Build() + + err := stagePGTDEVaultCredentials(ctx, k8s, execRecorder(&calls, nil), + "ns1", vault, newPods("a", "b"), naming.ContainerDatabase, tokenPath, caPath) + + assert.ErrorContains(t, err, "token secret") + assert.Equal(t, len(calls), 0, + "the Secrets are read before anything is written to any Pod") + }) + + t.Run("MissingKey", func(t *testing.T) { + var calls []execCall + k8s := fake.NewClientBuilder().WithObjects(secret).Build() + + badKey := tdeVaultSpec() + badKey.TokenSecret.Key = "nope" + + err := stagePGTDEVaultCredentials(ctx, k8s, execRecorder(&calls, nil), + "ns1", badKey, newPods("a"), naming.ContainerDatabase, tokenPath, caPath) + + assert.ErrorContains(t, err, `key "nope" not found`) + assert.Equal(t, len(calls), 0) + }) + + t.Run("FailureNamesThePod", func(t *testing.T) { + var calls []execCall + k8s := fake.NewClientBuilder().WithObjects(secret).Build() + + err := stagePGTDEVaultCredentials(ctx, k8s, + execRecorder(&calls, func(call execCall) error { + if call.pod == "b" { + return errors.New("no space left on device") + } + return nil + }), + "ns1", vault, newPods("a", "b", "c"), naming.ContainerDatabase, + tokenPath, caPath) + + assert.ErrorContains(t, err, "pod b") + assert.ErrorContains(t, err, "no space left on device") + assert.Equal(t, len(calls), 3, + "staging stops at the first instance it cannot write to") + }) +} + +func TestWriteTempFile(t *testing.T) { + t.Parallel() + ctx := context.Background() + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "pgc1-instance1-abcd-0"}, + } + + t.Run("PipesDataAndSetsMode", func(t *testing.T) { + var calls []execCall + + assert.NilError(t, writeTempFile(ctx, execRecorder(&calls, nil), pod, + naming.ContainerDatabase, pgtde.TempTokenPath, []byte("hvs.sometoken"))) + + assert.Equal(t, len(calls), 1) + assert.Equal(t, calls[0].namespace, "ns1") + assert.Equal(t, calls[0].pod, pod.Name) + assert.Equal(t, calls[0].container, naming.ContainerDatabase) + assert.Equal(t, calls[0].stdin, "hvs.sometoken", + "the secret value should be piped in, not interpolated into the command") + assert.DeepEqual(t, calls[0].command[:2], []string{"bash", "-ceu"}) + assert.Assert(t, strings.Contains(calls[0].command[2], pgtde.TempTokenPath)) + assert.Assert(t, strings.Contains(calls[0].command[2], "umask 077"), + "the token file must never exist in a world readable state") + }) + + t.Run("ShortWrite", func(t *testing.T) { + // The container reports fewer bytes on disk than were sent. + err := writeTempFile(ctx, + func(ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + _, _ = io.WriteString(stdout, "4\n") + return nil + }, + pod, naming.ContainerDatabase, pgtde.TempTokenPath, []byte("hvs.sometoken")) + + assert.ErrorContains(t, err, "wrote 4 of 13 bytes", + "a truncated token must not be accepted as written") + }) + + t.Run("ExecFails", func(t *testing.T) { + var calls []execCall + + err := writeTempFile(ctx, + execRecorder(&calls, func(execCall) error { + return errors.New("no such file or directory") + }), + pod, naming.ContainerDatabase, pgtde.TempTokenPath, []byte("x")) + + assert.ErrorContains(t, err, pgtde.TempTokenPath) + assert.ErrorContains(t, err, "no such file or directory") + }) +} + +// countingReader counts Get calls made against the embedded reader. +type countingReader struct { + client.Reader + gets *int +} + +func (c *countingReader) Get( + ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption, +) error { + *c.gets++ + return c.Reader.Get(ctx, key, obj, opts...) +} + +func TestReconcilePGTDEProviders(t *testing.T) { + t.Parallel() + ctx := context.Background() + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "vault-secret"}, + Data: map[string][]byte{ + "token": []byte("hvs.newtoken"), + "ca.crt": []byte("-----BEGIN CERTIFICATE-----"), + }, + } + + vault := tdeVaultSpec() + tokenPath, caPath := pgtde.VaultCredentialPaths(vault) + tempTokenPath, tempCAPath := pgtde.TempVaultCredentialPaths(vault) + + standardRevision, err := pgtde.VaultRevision(vault, tokenPath, caPath) + assert.NilError(t, err) + tempRevision, err := pgtde.VaultRevision(vault, tempTokenPath, tempCAPath) + assert.NilError(t, err) + + newCluster := func() *v1beta1.PostgresCluster { + cluster := &v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "pgc1", UID: "the-uid"}, + } + cluster.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + Vault: tdeVaultSpec(), + } + return cluster + } + + // psqlCalls returns the subset of calls that ran SQL, ignoring the shell + // calls used to write and remove temporary files. + psqlCalls := func(calls []execCall) []execCall { + var out []execCall + for _, call := range calls { + if len(call.command) > 0 && call.command[0] == "psql" { + out = append(out, call) + } + } + return out + } + + t.Run("Disabled", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Spec.Extensions.PGTDE.Enabled = false + cluster.Status.PGTDERevision = standardRevision + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, cluster.Status.PGTDERevision, "", + "the revision must be cleared so re-enabling starts from scratch") + }) + + t.Run("NoVaultSpec", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Spec.Extensions.PGTDE.Vault = nil + cluster.Status.PGTDERevision = standardRevision + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, cluster.Status.PGTDERevision, "") + }) + + t.Run("WaitsForRollout", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + + instance := tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}) + // The Pod is running an older revision than the StatefulSet intends. + instance.Pods[0].Labels[appsv1.StatefulSetRevisionLabel] = "rev-0" + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{instance}} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, len(calls), 0, + "SQL must not run against a Pod that is mid-rollout") + assert.Equal(t, cluster.Status.PGTDERevision, "") + }) + + t.Run("WaitsForOtherInstances", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + + primary := tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}) + replica := tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}) + replica.Name = "instance2-efgh" + replica.Pods[0].Name = "pgc1-instance2-efgh-0" + replica.Pods[0].Annotations["status"] = `{"role":"replica"}` + replica.Pods[0].Labels[appsv1.StatefulSetRevisionLabel] = "rev-0" + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{primary, replica}} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, len(calls), 0, + "every instance must match its template, not just the primary") + }) + + t.Run("NoWritablePod", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + + instance := tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}) + instance.Pods[0].Annotations["status"] = `{"role":"replica"}` + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{instance}} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, len(calls), 0) + }) + + t.Run("WaitsForExtension", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + // No TDEInstalledAnnotation: the Pod predates the extension install. + observed := &observedInstances{forCluster: []*Instance{tdeInstance(nil)}} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, len(calls), 0, + "the provider cannot be configured before pg_tde is loaded") + assert.Equal(t, cluster.Status.PGTDERevision, "") + }) + + t.Run("AlreadyConfigured", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Status.PGTDERevision = standardRevision + // Steady state: the revision matches and the last reconcile confirmed + // the data volumes hold no staged credentials. + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: v1beta1.PGTDEVaultProviderReady, + Status: metav1.ConditionTrue, + Reason: "Configured", + }) + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, failPatch(t))) + assert.Equal(t, len(calls), 0, "a matching revision is a no-op") + }) + + // A change that fails after staging leaves the new vault token on every + // data volume. Reverting the spec makes the stored revision match again, + // so nothing else in this function will ever look at those files. + t.Run("RevertedAfterFailedChange", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Status.PGTDERevision = standardRevision + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: v1beta1.PGTDEVaultProviderReady, + Status: metav1.ConditionFalse, + Reason: "ChangeFailed", + Message: "permission denied", + }) + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + patched := 0 + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { patched++; return nil })) + + assert.Equal(t, len(calls), 1, + "the credentials staged for the abandoned change must be removed") + assert.Assert(t, strings.Contains(calls[0].command[2], pgtde.TempTokenPath)) + assert.Assert(t, strings.Contains(calls[0].command[2], pgtde.TempCAPath)) + + // The failure the user already resolved must stop being reported. + assertTDEProviderCondition(t, cluster, metav1.ConditionTrue, "Configured") + assert.Equal(t, patched, 1) + }) + + // The revision is only ever stored alongside a condition, so a missing one + // means the status was lost rather than that the volumes are known clean. + t.Run("RevisionWithoutCondition", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Status.PGTDERevision = standardRevision + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { return nil })) + + assertTDEProviderCondition(t, cluster, metav1.ConditionTrue, "Configured") + }) + + t.Run("InitialSetup", func(t *testing.T) { + var calls []execCall + patched := 0 + cluster := newCluster() + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { patched++; return nil })) + + sql := psqlCalls(calls) + assert.Equal(t, len(sql), 3, + "initial setup adds the provider, creates the key and sets it as default") + assert.Assert(t, strings.Contains(sql[0].stdin, "pg_tde_add_global_key_provider_vault_v2")) + assert.Assert(t, strings.Contains(sql[1].stdin, "pg_tde_create_key_using_global_key_provider")) + assert.Assert(t, strings.Contains(sql[2].stdin, "pg_tde_set_default_key_using_global_key_provider")) + + assert.Assert(t, argsContain(sql[0].command, "--set=token_path="+tokenPath), + "initial setup uses the mounted credential paths, not the temporary ones") + assert.Assert(t, argsContain(sql[0].command, "--set=ca_path="+caPath)) + + assert.Equal(t, len(calls), len(sql), + "no temporary files are needed when there is nothing to rotate") + assert.Equal(t, cluster.Status.PGTDERevision, standardRevision) + assert.Equal(t, patched, 1, "the revision must be persisted immediately") + assertTDEProviderCondition(t, cluster, metav1.ConditionTrue, "Configured") + }) + + t.Run("PhaseOne", func(t *testing.T) { + var calls []execCall + patched := 0 + cluster := newCluster() + // A revision that is neither the standard nor the temp one: the user + // changed the vault configuration. + cluster.Status.PGTDERevision = "stale" + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { patched++; return nil })) + + // The new credentials are staged on the persistent volume first... + assert.Equal(t, len(calls), 3) + assert.Assert(t, strings.Contains(calls[0].command[2], tempTokenPath)) + assert.Equal(t, calls[0].stdin, "hvs.newtoken") + assert.Assert(t, strings.Contains(calls[1].command[2], tempCAPath)) + + // ...and only then is the provider pointed at them. + sql := psqlCalls(calls) + assert.Equal(t, len(sql), 1) + assert.Assert(t, strings.Contains(sql[0].stdin, "pg_tde_change_global_key_provider_vault_v2")) + assert.Assert(t, argsContain(sql[0].command, "--set=token_path="+tempTokenPath)) + assert.Assert(t, argsContain(sql[0].command, "--set=ca_path="+tempCAPath)) + + assert.Equal(t, cluster.Status.PGTDERevision, tempRevision, + "the temp revision releases the volume hold in reconcileInstance") + assert.Equal(t, patched, 1) + assertTDEProviderCondition(t, cluster, metav1.ConditionFalse, "ChangeInProgress") + }) + + t.Run("PhaseOneSecretMissing", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Status.PGTDERevision = "stale" + + r := &Reconciler{ + Client: fake.NewClientBuilder().Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + patched := 0 + err := r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { patched++; return nil }) + assert.ErrorContains(t, err, "token secret") + assert.Equal(t, len(psqlCalls(calls)), 0, + "the provider must not be changed to paths that were never written") + assert.Equal(t, cluster.Status.PGTDERevision, "stale", + "the revision must not advance when phase one fails") + + // reconcileInstance keeps holding the old vault volume while the + // revision is stale, so the reason must reach the user. + assertTDEProviderCondition(t, cluster, metav1.ConditionFalse, "ChangeFailed") + assert.Assert(t, strings.Contains(tdeCondition(cluster).Message, "vault-secret"), + "the condition should name the Secret that could not be read") + assert.Equal(t, patched, 1, + "the failure condition is useless unless it is written to the API") + assertEvent(t, r.Recorder, "PGTDEVaultProviderChangeFailed") + }) + + t.Run("PhaseTwo", func(t *testing.T) { + var calls []execCall + patched := 0 + cluster := newCluster() + cluster.Status.PGTDERevision = tempRevision + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { patched++; return nil })) + + sql := psqlCalls(calls) + assert.Equal(t, len(sql), 1) + assert.Assert(t, strings.Contains(sql[0].stdin, "pg_tde_change_global_key_provider_vault_v2")) + assert.Assert(t, argsContain(sql[0].command, "--set=token_path="+tokenPath), + "phase two points the provider back at the mounted paths") + assert.Assert(t, argsContain(sql[0].command, "--set=ca_path="+caPath)) + + // The staged credentials are removed once nothing references them. + assert.Equal(t, len(calls), 2) + assert.Assert(t, strings.Contains(calls[1].command[2], tempTokenPath)) + assert.Assert(t, strings.Contains(calls[1].command[2], tempCAPath)) + + assert.Equal(t, cluster.Status.PGTDERevision, standardRevision) + assert.Equal(t, patched, 1) + assertTDEProviderCondition(t, cluster, metav1.ConditionTrue, "Configured") + }) + + t.Run("PhaseTwoKeepsTempFilesOnFailure", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + cluster.Status.PGTDERevision = tempRevision + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, func(call execCall) error { + if call.command[0] == "psql" { + return errors.New("could not connect to server") + } + return nil + }), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + err := r.reconcilePGTDEProviders(ctx, cluster, observed, func() error { return nil }) + assert.ErrorContains(t, err, "could not connect to server") + assert.Equal(t, len(calls), 1, + "temp files must survive a failed phase two so it can be retried") + assert.Equal(t, cluster.Status.PGTDERevision, tempRevision) + assertTDEProviderCondition(t, cluster, metav1.ConditionFalse, "ChangeFailed") + }) + + t.Run("CleanupRetrySucceeds", func(t *testing.T) { + var calls []execCall + fail := true + cluster := newCluster() + cluster.Status.PGTDERevision = standardRevision + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, func(execCall) error { + if fail { + return errors.New("permission denied") + } + return nil + }), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + fail = false + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { return nil })) + assert.Equal(t, len(calls), 1) + assert.Equal(t, len(psqlCalls(calls)), 0, "nothing about the provider changed") + assertTDEProviderCondition(t, cluster, metav1.ConditionTrue, "Configured") + }) + + t.Run("PatchStatusFails", func(t *testing.T) { + var calls []execCall + cluster := newCluster() + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + observed := &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + }} + + err := r.reconcilePGTDEProviders(ctx, cluster, observed, + func() error { return errors.New("conflict") }) + assert.ErrorContains(t, err, "patch status") + }) +} + +// tdeCondition returns the PGTDEVaultProviderReady condition, failing the test +// when it is absent. +func tdeCondition(cluster *v1beta1.PostgresCluster) metav1.Condition { + condition := meta.FindStatusCondition(cluster.Status.Conditions, + v1beta1.PGTDEVaultProviderReady) + if condition == nil { + return metav1.Condition{Reason: ""} + } + return *condition +} + +// assertTDEProviderCondition checks the status and reason of the +// PGTDEVaultProviderReady condition. +func assertTDEProviderCondition( + t *testing.T, cluster *v1beta1.PostgresCluster, + status metav1.ConditionStatus, reason string, +) { + t.Helper() + + condition := tdeCondition(cluster) + assert.Equal(t, string(condition.Status), string(status)) + assert.Equal(t, condition.Reason, reason) +} + +// assertEvent checks that an event with the given reason was recorded. +func assertEvent(t *testing.T, recorder record.EventRecorder, reason string) { + t.Helper() + + rec, ok := recorder.(*events.Recorder) + assert.Assert(t, ok, "expected a testing recorder") + for _, event := range rec.Events { + if event.Reason == reason { + return + } + } + t.Errorf("expected an event with reason %q, got %v", reason, rec.Events) +} + +// failPatch returns a patch function that fails the test when it is called. +func failPatch(t *testing.T) func() error { + t.Helper() + return func() error { + t.Error("status should not be patched") + return nil + } +} + +// argsContain reports whether command contains arg. +func argsContain(command []string, arg string) bool { + for _, c := range command { + if c == arg { + return true + } + } + return false +} + +func TestReconcilePostgresDatabasesPGTDEReporting(t *testing.T) { + t.Parallel() + ctx := context.Background() + + // newCluster returns a cluster that reconcilePostgresDatabases will act on. + newCluster := func(enabled bool) *v1beta1.PostgresCluster { + cluster := &v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", Name: "pgc1", UID: "the-uid", + Labels: map[string]string{naming.LabelVersion: "17.2.0"}, + }, + } + cluster.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{Enabled: enabled} + return cluster + } + + observed := func() *observedInstances { + return &observedInstances{forCluster: []*Instance{tdeInstance(nil)}} + } + + // pgTDECondition reports the PGTDEEnabled condition, which decides whether + // pg_tde goes into shared_preload_libraries and whether the vault volume is + // mounted. + pgTDECondition := func(cluster *v1beta1.PostgresCluster) *metav1.Condition { + return meta.FindStatusCondition(cluster.Status.Conditions, v1beta1.PGTDEEnabled) + } + + t.Run("InstallFailureIsNotReportedAsEnabled", func(t *testing.T) { + cluster := newCluster(true) + calls := 0 + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: func(ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + if strings.Contains(string(b), "pg_tde") { + calls++ + return errors.New("could not open extension control file") + } + return nil + }, + } + + // The failure is folded into the "OK" flags rather than returned, the + // same way the other extensions handle theirs; it withholds the + // DatabaseRevision so the next reconcile retries. + patched := 0 + assert.NilError(t, r.reconcilePostgresDatabases(ctx, cluster, observed(), + func() error { patched++; return nil })) + assert.Equal(t, calls, 1, "only the real run reaches PodExec") + assert.Equal(t, patched, 1, "the failure event and condition still need persisting") + + // The regression: the hash run executes the same statements against a + // fake executor that cannot fail, and used to flip this to True before + // CREATE EXTENSION had been attempted. + condition := pgTDECondition(cluster) + assert.Assert(t, condition == nil || condition.Status != metav1.ConditionTrue, + "pg_tde must not be reported as enabled when CREATE EXTENSION failed") + assertEvent(t, r.Recorder, "PGTDEInstallFailed") + assert.Equal(t, cluster.Status.DatabaseRevision, "") + }) + + t.Run("SuccessIsReported", func(t *testing.T) { + cluster := newCluster(true) + patched := 0 + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: func(ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + return nil + }, + } + + assert.NilError(t, r.reconcilePostgresDatabases(ctx, cluster, observed(), + func() error { patched++; return nil })) + + condition := pgTDECondition(cluster) + assert.Assert(t, condition != nil) + assert.Equal(t, condition.Status, metav1.ConditionTrue) + assert.Assert(t, cluster.Status.DatabaseRevision != "") + assert.Equal(t, patched, 1, "the condition and the revision share one patch") + }) + + t.Run("NothingIsReportedWithoutAWritablePod", func(t *testing.T) { + cluster := newCluster(true) + + instance := tdeInstance(nil) + instance.Pods[0].Annotations["status"] = `{"role":"replica"}` + + r := &Reconciler{Recorder: events.NewRecorder(t, runtime.Scheme)} + + assert.NilError(t, r.reconcilePostgresDatabases(ctx, cluster, + &observedInstances{forCluster: []*Instance{instance}}, failPatch(t))) + assert.Assert(t, pgTDECondition(cluster) == nil, + "no SQL ran, so there is nothing to report") + }) + + t.Run("DisableIsReportedOnlyAfterItRuns", func(t *testing.T) { + cluster := newCluster(false) + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: func(ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + if strings.Contains(string(b), "pg_tde") { + return errors.New("cannot drop extension") + } + return nil + }, + } + + assert.NilError(t, r.reconcilePostgresDatabases(ctx, cluster, observed(), + func() error { return nil })) + assert.Equal(t, cluster.Status.DatabaseRevision, "", "the drop must be retried") + + // Reporting False here would strip pg_tde from shared_preload_libraries + // and restart the Pods while the extension is still installed. + assert.Assert(t, pgTDECondition(cluster) == nil, + "a failed DROP EXTENSION must not be reported as disabled") + assertEvent(t, r.Recorder, "PGTDEDisableFailed") + }) +} + +func TestReconcilePostgresDatabasesPGTDEStatusIsIndependent(t *testing.T) { + t.Parallel() + ctx := context.Background() + + newCluster := func() *v1beta1.PostgresCluster { + cluster := &v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", Name: "pgc1", UID: "the-uid", + Labels: map[string]string{naming.LabelVersion: "17.2.0"}, + }, + } + cluster.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{Enabled: true} + return cluster + } + + observed := func() *observedInstances { + return &observedInstances{forCluster: []*Instance{tdeInstance(nil)}} + } + + // failOn returns a PodExec that fails every statement mentioning marker. + failOn := func(marker string) func( + ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + return func(ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + if strings.Contains(string(b), marker) { + return errors.New(marker + " is unavailable") + } + return nil + } + } + + t.Run("PersistedWhenAnotherExtensionFails", func(t *testing.T) { + cluster := newCluster() + patched := 0 + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: failOn("pgaudit"), + } + + assert.NilError(t, r.reconcilePostgresDatabases(ctx, cluster, observed(), + func() error { patched++; return nil })) + + // pgaudit failing withholds the revision, which is what makes the + // whole batch of SQL run again next time. + assert.Equal(t, cluster.Status.DatabaseRevision, "") + + // pg_tde installed fine, so its condition must not be held hostage: + // reconcilePGTDEProviders runs next and needs it. + condition := meta.FindStatusCondition(cluster.Status.Conditions, v1beta1.PGTDEEnabled) + assert.Assert(t, condition != nil, + "the pg_tde condition must not wait on unrelated extensions") + assert.Equal(t, condition.Status, metav1.ConditionTrue) + assert.Equal(t, patched, 1, "the condition should be persisted on its own") + }) + + t.Run("PatchFailureDoesNotCancelTheReconcile", func(t *testing.T) { + cluster := newCluster() + + r := &Reconciler{ + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: func(ctx context.Context, namespace, pod, container string, + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + return nil + }, + } + + // A conflict on the status subresource says nothing about whether the + // SQL worked, and Reconcile patches again before it returns. + err := r.reconcilePostgresDatabases(ctx, cluster, observed(), + func() error { return errors.New("the object has been modified") }) + + assert.NilError(t, err, + "a status conflict must not stop reconcilePGTDEProviders from running") + assert.Assert(t, cluster.Status.DatabaseRevision != "", + "the revision is still correct in memory for the final patch") + }) +} + +func TestReconcilePGTDEProvidersMultipleInstances(t *testing.T) { + t.Parallel() + ctx := context.Background() + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "vault-secret"}, + Data: map[string][]byte{ + "token": []byte("hvs.newtoken"), + "ca.crt": []byte("-----BEGIN CERTIFICATE-----"), + }, + } + + vault := tdeVaultSpec() + tokenPath, caPath := pgtde.VaultCredentialPaths(vault) + tempTokenPath, tempCAPath := pgtde.TempVaultCredentialPaths(vault) + + standardRevision, err := pgtde.VaultRevision(vault, tokenPath, caPath) + assert.NilError(t, err) + tempRevision, err := pgtde.VaultRevision(vault, tempTokenPath, tempCAPath) + assert.NilError(t, err) + + newCluster := func(revision string) *v1beta1.PostgresCluster { + cluster := &v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1", Name: "pgc1", UID: "the-uid"}, + } + cluster.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{Enabled: true, Vault: tdeVaultSpec()} + cluster.Status.PGTDERevision = revision + return cluster + } + + // replica returns a running, up-to-date instance that is not writable. + replica := func(name string) *Instance { + instance := tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}) + instance.Name = name + instance.Pods[0].Name = "pgc1-" + name + "-0" + instance.Pods[0].Annotations["status"] = `{"role":"replica"}` + return instance + } + + // threeInstances returns a primary plus two replicas, all healthy. + threeInstances := func() *observedInstances { + return &observedInstances{forCluster: []*Instance{ + tdeInstance(map[string]string{naming.TDEInstalledAnnotation: "true"}), + replica("instance2-efgh"), + replica("instance3-ijkl"), + }} + } + + podsWritten := func(calls []execCall, marker string) []string { + var names []string + for _, call := range calls { + if len(call.command) > 2 && strings.Contains(call.command[2], marker) { + names = append(names, call.pod) + } + } + return names + } + + t.Run("PhaseOneStagesOnEveryInstance", func(t *testing.T) { + var calls []execCall + cluster := newCluster("stale") + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, threeInstances(), + func() error { return nil })) + + // pg_tde resolves one path cluster-wide; a replica promoted before + // phase 2 must find the credentials on its own volume. + assert.DeepEqual(t, podsWritten(calls, tempTokenPath), []string{ + "pgc1-instance1-abcd-0", "pgc1-instance2-efgh-0", "pgc1-instance3-ijkl-0", + }) + assert.DeepEqual(t, podsWritten(calls, tempCAPath), []string{ + "pgc1-instance1-abcd-0", "pgc1-instance2-efgh-0", "pgc1-instance3-ijkl-0", + }) + + // The SQL still runs once, on the primary. + var sql []execCall + for _, call := range calls { + if call.command[0] == "psql" { + sql = append(sql, call) + } + } + assert.Equal(t, len(sql), 1) + assert.Equal(t, sql[0].pod, "pgc1-instance1-abcd-0") + assert.Equal(t, cluster.Status.PGTDERevision, tempRevision) + }) + + t.Run("PhaseOneWaitsForEveryInstance", func(t *testing.T) { + var calls []execCall + cluster := newCluster("stale") + + // One replica's database container is not running yet. + instances := threeInstances() + instances.forCluster[2].Pods[0].Status.ContainerStatuses[0].State = + corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "PodInitializing"}} + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + + patched := 0 + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, instances, + func() error { patched++; return nil })) + + assert.Equal(t, len(calls), 0, + "the provider must not name a path that one instance cannot read") + assert.Equal(t, cluster.Status.PGTDERevision, "stale", + "the volume hold stays until every instance is staged") + assertTDEProviderCondition(t, cluster, metav1.ConditionFalse, "WaitingForInstances") + assert.Equal(t, patched, 1, "the reason for waiting must be visible") + }) + + t.Run("PhaseTwoCleansEveryInstance", func(t *testing.T) { + var calls []execCall + cluster := newCluster(tempRevision) + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, threeInstances(), + func() error { return nil })) + + assert.DeepEqual(t, podsWritten(calls, "rm -f"), []string{ + "pgc1-instance1-abcd-0", "pgc1-instance2-efgh-0", "pgc1-instance3-ijkl-0", + }) + assert.Equal(t, cluster.Status.PGTDERevision, standardRevision) + assertTDEProviderCondition(t, cluster, metav1.ConditionTrue, "Configured") + }) + + t.Run("PhaseTwoCleanupIncompleteIsRetried", func(t *testing.T) { + var calls []execCall + cluster := newCluster(tempRevision) + + // A replica is down, so its copy of the token cannot be removed. + instances := threeInstances() + instances.forCluster[2].Pods[0].Status.ContainerStatuses[0].State = + corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}} + + r := &Reconciler{ + Client: fake.NewClientBuilder().WithObjects(secret).Build(), + Recorder: events.NewRecorder(t, runtime.Scheme), + PodExec: execRecorder(&calls, nil), + } + + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, instances, + func() error { return nil })) + assert.Equal(t, cluster.Status.PGTDERevision, tempRevision) + + instances = threeInstances() + assert.NilError(t, r.reconcilePGTDEProviders(ctx, cluster, instances, + func() error { return nil })) + assert.Equal(t, cluster.Status.PGTDERevision, standardRevision) + }) +} + +func TestPGTDEVaultChangeFor(t *testing.T) { + t.Parallel() + + vault := tdeVaultSpec() + tokenPath, caPath := pgtde.VaultCredentialPaths(vault) + tempTokenPath, tempCAPath := pgtde.TempVaultCredentialPaths(vault) + + standardRevision, err := pgtde.VaultRevision(vault, tokenPath, caPath) + assert.NilError(t, err) + tempRevision, err := pgtde.VaultRevision(vault, tempTokenPath, tempCAPath) + assert.NilError(t, err) + + clusterWith := func(revision string) *v1beta1.PostgresCluster { + cluster := &v1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + Vault: tdeVaultSpec(), + } + cluster.Status.PGTDERevision = revision + return cluster + } + + for _, tc := range []struct { + name string + revision string + expected pgtde.Phase + }{ + {"InitialSetup", "", pgtde.InitialSetup}, + {"Configured", standardRevision, pgtde.Configured}, + {"Finalize", tempRevision, pgtde.Finalize}, + {"StageCredentials", "a-revision-from-another-vault", pgtde.StageCredentials}, + } { + t.Run(tc.name, func(t *testing.T) { + change, err := pgtde.VaultChangeFor(clusterWith(tc.revision)) + assert.NilError(t, err) + assert.Equal(t, change.Phase, tc.expected) + + assert.Equal(t, change.TokenPath, tokenPath) + assert.Equal(t, change.CAPath, caPath) + assert.Equal(t, change.TempTokenPath, tempTokenPath) + assert.Equal(t, change.TempCAPath, tempCAPath) + assert.Equal(t, change.StandardRevision, standardRevision) + assert.Equal(t, change.TempRevision, tempRevision) + }) + } + + // reconcileInstance holds the Pods' vault volume in exactly one phase. + // Holding in any other one pins the StatefulSet to credentials the provider + // has already moved off of, and the Pods never roll. + t.Run("HoldsTheVolumeInOnePhase", func(t *testing.T) { + held := map[pgtde.Phase]bool{} + for _, revision := range []string{ + "", standardRevision, tempRevision, "a-revision-from-another-vault", + } { + change, err := pgtde.VaultChangeFor(clusterWith(revision)) + assert.NilError(t, err) + held[change.Phase] = change.Phase == pgtde.StageCredentials + } + + assert.DeepEqual(t, held, map[pgtde.Phase]bool{ + pgtde.InitialSetup: false, + pgtde.Configured: false, + pgtde.StageCredentials: true, + pgtde.Finalize: false, + }) + }) +} diff --git a/internal/controller/postgrescluster/postgres.go b/internal/controller/postgrescluster/postgres.go index 8811c2f25f..eb5d9cd958 100644 --- a/internal/controller/postgrescluster/postgres.go +++ b/internal/controller/postgrescluster/postgres.go @@ -13,6 +13,7 @@ import ( "net/url" "regexp" "sort" + "strconv" "strings" gover "github.com/hashicorp/go-version" @@ -20,6 +21,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -28,6 +30,7 @@ import ( "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/percona/percona-postgresql-operator/v2/internal/controller/runtime" "github.com/percona/percona-postgresql-operator/v2/internal/feature" "github.com/percona/percona-postgresql-operator/v2/internal/initialize" "github.com/percona/percona-postgresql-operator/v2/internal/logging" @@ -37,6 +40,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/pgrepack" "github.com/percona/percona-postgresql-operator/v2/internal/pgstatmonitor" "github.com/percona/percona-postgresql-operator/v2/internal/pgstatstatements" + "github.com/percona/percona-postgresql-operator/v2/internal/pgtde" "github.com/percona/percona-postgresql-operator/v2/internal/pgvector" "github.com/percona/percona-postgresql-operator/v2/internal/postgis" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" @@ -193,7 +197,10 @@ func (r *Reconciler) generatePostgresUserSecret( // reconcilePostgresDatabases creates databases inside of PostgreSQL. func (r *Reconciler) reconcilePostgresDatabases( - ctx context.Context, cluster *v1beta1.PostgresCluster, instances *observedInstances, + ctx context.Context, + cluster *v1beta1.PostgresCluster, + instances *observedInstances, + patchStatus func() error, ) error { const container = naming.ContainerDatabase var podExecutor postgres.Executor @@ -250,8 +257,15 @@ func (r *Reconciler) reconcilePostgresDatabases( } // Calculate a hash of the SQL that should be executed in PostgreSQL. - // K8SPG-375, K8SPG-577, K8SPG-699 - var pgAuditOK, pgStatMonitorOK, pgStatStatementsOK, pgvectorOK, pgRepackOK, pgCronOK, setUserOK, postgisInstallOK bool + // K8SPG-375, K8SPG-577, K8SPG-699, K8SPG-911 + var pgAuditOK, pgStatMonitorOK, pgStatStatementsOK, pgvectorOK, pgRepackOK, pgCronOK, setUserOK, pgTdeOK, postgisInstallOK bool + + // K8SPG-911: create runs twice, once against a fake executor to hash the + // statements and once for real. Only the real run may be reported, so its + // result is kept separately from the flag above. + var pgTdeRan bool + var pgTdeErr error + create := func(ctx context.Context, exec postgres.Executor) error { // validate version string before running it in database _, err := gover.NewVersion(cluster.Labels[naming.LabelVersion]) @@ -362,6 +376,10 @@ func (r *Reconciler) reconcilePostgresDatabases( } } + // K8SPG-911 + pgTdeRan, pgTdeErr = true, pgtde.ReconcileExtension(ctx, exec, cluster) + pgTdeOK = pgTdeErr == nil + // Enabling PostGIS extensions is a one-way operation // e.g., you can take a PostgresCluster and turn it into a PostGISCluster, // but you cannot reverse the process, as that would potentially remove an extension @@ -378,7 +396,7 @@ func (r *Reconciler) reconcilePostgresDatabases( } // Calculate a hash of the SQL that should be executed in PostgreSQL. - revision, err := safeHash32(func(hasher io.Writer) error { + revision, err := util.SafeHash32(func(hasher io.Writer) error { // Discard log messages about executing SQL. return create(logging.NewContext(ctx, logging.Discard()), func( _ context.Context, stdin io.Reader, _, _ io.Writer, command ...string, @@ -401,19 +419,414 @@ func (r *Reconciler) reconcilePostgresDatabases( // Apply the necessary SQL and record its hash in cluster.Status. Include // the hash in any log messages. - if err == nil { + // Forget what the hash run observed; those statements were never sent + // to PostgreSQL. + pgTdeRan, pgTdeErr = false, nil + log := logging.FromContext(ctx).WithValues("revision", revision) err = errors.WithStack(create(logging.NewContext(ctx, log), podExecutor)) } + + // K8SPG-911: report pg_tde only when it really ran. The PGTDEEnabled + // condition gates shared_preload_libraries and the vault volume, so + // claiming the extension exists before CREATE EXTENSION has succeeded + // sends the operator on to configure a key provider that cannot work. + // + // This is deliberately outside the revision guard below. Whether pg_tde is + // installed is a fact about pg_tde alone, and reconcilePGTDEProviders acts + // on it later in this same reconcile; making it wait for every other + // extension to succeed would let an unrelated failure, such as a missing + // pgvector library, stall the key provider setup indefinitely. + if pgTdeRan { + pgtde.ReportExtension(cluster, r.Recorder, pgTdeErr) + } + // K8SPG-472 - if err == nil && pgStatMonitorOK && pgAuditOK && pgvectorOK && postgisInstallOK && pgRepackOK && pgCronOK && setUserOK { + allExtensionsOK := err == nil && + pgStatMonitorOK && + pgAuditOK && + pgvectorOK && + postgisInstallOK && + pgRepackOK && + pgCronOK && + setUserOK && + pgTdeOK + + if allExtensionsOK { + // Every statement above succeeded, so none of them need to run again. cluster.Status.DatabaseRevision = revision } + if pgTdeRan || allExtensionsOK { + if patchErr := patchStatus(); patchErr != nil { + // Losing this patch only costs a repeat of the SQL above, all of + // which is idempotent, and Reconcile patches the status again on + // its way out. Canceling the reconcilers that follow, including + // the pg_tde key provider setup, would cost more. + logging.FromContext(ctx).Error(patchErr, "failed to patch cluster status") + } + } + return err } +// reconcilePGTDEProviders configures pg_tde providers using a two-phase +// approach for vault credential changes: +// +// - Phase 1: The pod still mounts the OLD vault secret. Fetch the new +// credentials from the Kubernetes Secret into temp files and run +// pg_tde_change_global_key_provider_vault_v2 with those temp paths. +// Store a "temp" revision (hash includes temp paths). This releases +// the volume hold so the StatefulSet updates and pods restart. +// +// - Phase 2: After restart, the pod mounts the NEW vault secret at +// standard paths. Run pg_tde_change_global_key_provider_vault_v2 +// again with the standard mount paths so pg_tde no longer references +// temp files. Store the "standard" revision. +func (r *Reconciler) reconcilePGTDEProviders( + ctx context.Context, + cluster *v1beta1.PostgresCluster, + instances *observedInstances, + patchStatus func() error, +) error { + const container = naming.ContainerDatabase + + log := logging.FromContext(ctx).WithName("PGTDE") + + if !cluster.Spec.Extensions.PGTDE.Enabled || cluster.Spec.Extensions.PGTDE.Vault == nil { + cluster.Status.PGTDERevision = "" + meta.RemoveStatusCondition(&cluster.Status.Conditions, v1beta1.PGTDEVaultProviderReady) + + return nil + } + + // Wait for all instances to match their pod templates before configuring + // the vault provider. This prevents running SQL on pods that are mid-rollout. + for _, inst := range instances.forCluster { + if matches, known := inst.PodMatchesPodTemplate(); !matches || !known { + log.V(1).Info("Waiting for instance to be updated", "instance", inst.Name) + return nil + } + } + + pod, _ := instances.writablePod(container) + if pod == nil { + log.V(1).Info("Waiting for a writable instance") + return nil + } + + pods, allRunning := instances.runningPods(container) + + // We need to configure pg_tde after volumes are mounted and extension is created + if _, ok := pod.Annotations[naming.TDEInstalledAnnotation]; !ok { + log.V(1).Info("Waiting for pg_tde to be installed", "pod", pod.Name) + return nil + } + + log = log.WithValues("pod", pod.Name) + ctx = logging.NewContext(ctx, log) + pgExecutor := func( + ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + return r.PodExec(ctx, pod.Namespace, pod.Name, container, stdin, stdout, stderr, command...) + } + + vault := cluster.Spec.Extensions.PGTDE.Vault + + change, err := pgtde.VaultChangeFor(cluster) + if err == nil && change.Phase == pgtde.Configured { + if !allRunning { + log.V(1).Info("Waiting for all pods to be running") + return nil + } + + condition := meta.FindStatusCondition(cluster.Status.Conditions, v1beta1.PGTDEVaultProviderReady) + if condition == nil || condition.Status != metav1.ConditionTrue { + r.setPGTDEVaultProviderCondition(cluster) + + if err := r.cleanupTempPGTDEFiles(ctx, pods, container); err != nil { + log.Error(err, "failed to remove staged pg_tde vault credentials", + "paths", []string{pgtde.TempTokenPath, pgtde.TempCAPath}) + } + + return patchStatus() + } + return nil + } + + var revision string + if err == nil { + switch change.Phase { + case pgtde.InitialSetup: + // No provider has been configured, so the Pods already mount the + // credentials in the spec and there is nothing to stage. + err = errors.WithStack(pgtde.ReconcileVaultProvider( + ctx, pgExecutor, cluster, change.TokenPath, change.CAPath)) + revision = change.StandardRevision + + case pgtde.Finalize: + if !allRunning { + log.V(1).Info("Waiting for all pods to be running before finalizing provider change") + return nil + } + + // Phase 2: pod restarted with new volume mounted at standard paths. + // Change provider from temp paths to persistent mount paths, then + // clean up the temp files from /pgdata. + log.Info("finalizing vault provider change with standard mount paths") + err = errors.WithStack(pgtde.ReconcileVaultProvider( + ctx, pgExecutor, cluster, change.TokenPath, change.CAPath)) + if err == nil { + if err := r.cleanupTempPGTDEFiles(ctx, pods, container); err != nil { + log.Error(err, "failed to remove staged pg_tde vault credentials", + "paths", []string{pgtde.TempTokenPath, pgtde.TempCAPath}) + } + } + revision = change.StandardRevision + + default: + // Phase 1: vault config changed, pods still have old credentials. + // Stage the new credentials in temp files on /pgdata (persistent + // volume) and change the provider to use those paths. The temp + // files survive the pod restart so pg_tde can read them until + // phase 2 runs. + // + // Every instance needs its own copy before the provider starts + // naming those paths. Staging on only some of them would leave the + // rest unable to resolve the key, and a replica promoted before + // phase 2 would come up without the credentials it needs. + if !allRunning { + log.Info("waiting for all instances to be running before staging vault credentials") + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: v1beta1.PGTDEVaultProviderReady, + Status: metav1.ConditionFalse, + Reason: "WaitingForInstances", + Message: "waiting for all instances to be running to stage vault credentials", + ObservedGeneration: cluster.GetGeneration(), + }) + return patchStatus() + } + + log.Info("changing vault provider using temporary credentials", "instances", len(pods)) + if err = stagePGTDEVaultCredentials(ctx, r.Client, r.PodExec, cluster.Namespace, + vault, pods, container, change.TempTokenPath, change.TempCAPath); err != nil { + break + } + + err = errors.WithStack(pgtde.ReconcileVaultProvider( + ctx, pgExecutor, cluster, change.TempTokenPath, change.TempCAPath)) + revision = change.TempRevision + } + } + + if err != nil { + // reconcileInstance holds the old vault volume until the revision + // advances, so a change that keeps failing pins the StatefulSet to + // the old credentials indefinitely. Surface the cause rather than + // leaving the user to guess why their Pods never roll. + r.Recorder.Event(cluster, corev1.EventTypeWarning, "PGTDEVaultProviderChangeFailed", err.Error()) + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: v1beta1.PGTDEVaultProviderReady, + Status: metav1.ConditionFalse, + Reason: "ChangeFailed", + Message: err.Error(), + ObservedGeneration: cluster.GetGeneration(), + }) + + // Report the failure even though the reconcile below returns an error; + // without this the condition is dropped along with the rest of the + // in-memory status changes. + if patchErr := patchStatus(); patchErr != nil { + log.Error(patchErr, "failed to report vault provider change failure") + } + + return err + } + + cluster.Status.PGTDERevision = revision + + if revision == change.TempRevision { + // Phase 1 done, phase 2 still pending: the provider currently points at + // the staged credentials, not at the ones in the spec. + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: v1beta1.PGTDEVaultProviderReady, + Status: metav1.ConditionFalse, + Reason: "ChangeInProgress", + Message: "waiting for Pods to restart with the new vault credentials", + ObservedGeneration: cluster.GetGeneration(), + }) + } else { + r.setPGTDEVaultProviderCondition(cluster) + } + + if err := patchStatus(); err != nil { + return errors.Wrap(err, "patch status") + } + + return nil +} + +// setPGTDEVaultProviderCondition reports a provider that matches the spec, noting +// whether the credentials staged during the change were removed afterwards. +func (r *Reconciler) setPGTDEVaultProviderCondition(cluster *v1beta1.PostgresCluster) { + condition := metav1.Condition{ + Type: v1beta1.PGTDEVaultProviderReady, + Status: metav1.ConditionTrue, + Reason: "Configured", + Message: "pg_tde vault key provider matches the spec", + ObservedGeneration: cluster.GetGeneration(), + } + + meta.SetStatusCondition(&cluster.Status.Conditions, condition) +} + +// cleanupTempPGTDEFiles removes the vault credentials staged during a provider +// change from every Pod's data volume. Each instance has its own volume, so +// each has its own copy to remove. When complete is false some instance could +// not be reached and its copy is still there, which counts as a failure so the +// caller retries. The error is returned rather than acted on because callers +// differ in how much they can do about it, but it is always logged and +// recorded: a token left behind sits in plaintext on a PersistentVolume until +// something removes it. +func (r *Reconciler) cleanupTempPGTDEFiles(ctx context.Context, pods []*corev1.Pod, container string) error { + var err error + + for _, pod := range pods { + // Both paths are removed unconditionally: the CA file may exist from + // an earlier configuration that used one even if the current spec + // does not. + if e := removeTempFiles(ctx, r.PodExec, pod, container, + pgtde.TempTokenPath, pgtde.TempCAPath); e != nil && err == nil { + err = errors.Wrapf(e, "pod %s", pod.Name) + } + } + + return err +} + +// stagePGTDEVaultCredentials copies the vault credentials out of their Secrets and +// into temporary files on every Pod's data volume. +// +// pg_tde's key provider configuration is cluster-wide: whatever path it names +// is the path every instance reads, including a replica that gets promoted +// while the change is in flight. Each instance has its own /pgdata, so the +// files have to exist on all of them, not just on the one running the SQL. +func stagePGTDEVaultCredentials( + ctx context.Context, + k8sClient client.Reader, + podExec runtime.PodExecutor, + namespace string, + vault *v1beta1.PGTDEVaultSpec, + pods []*corev1.Pod, + container string, // nolint:unparam + tokenPath, caPath string, +) error { + type stagedFile struct { + path string + data []byte + } + + token, err := pgTDESecretValue(ctx, k8sClient, namespace, vault.TokenSecret) + if err != nil { + return errors.Wrap(err, "token secret") + } + files := []stagedFile{{path: tokenPath, data: token}} + + if vault.HasCA() { + ca, err := pgTDESecretValue(ctx, k8sClient, namespace, vault.CASecret) + if err != nil { + return errors.Wrap(err, "CA secret") + } + files = append(files, stagedFile{path: caPath, data: ca}) + } + + for _, pod := range pods { + for _, file := range files { + if err := writeTempFile(ctx, podExec, pod, container, + file.path, file.data); err != nil { + return errors.Wrapf(err, "pod %s", pod.Name) + } + } + } + + return nil +} + +// pgTDESecretValue reads one key from PGTDESecretObjectReference. +func pgTDESecretValue( + ctx context.Context, + k8sClient client.Reader, + namespace string, + secretRef v1beta1.PGTDESecretObjectReference, +) ([]byte, error) { + secret := &corev1.Secret{} + if err := k8sClient.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: secretRef.Name, + }, secret); err != nil { + return nil, errors.Wrapf(err, "get secret %q", secretRef.Name) + } + + data, ok := secret.Data[secretRef.Key] + if !ok { + return nil, errors.Errorf("key %q not found in secret %q", secretRef.Key, secretRef.Name) + } + + return data, nil +} + +// writeTempFile writes data to a file inside a pod container. +func writeTempFile( + ctx context.Context, + podExec runtime.PodExecutor, + pod *corev1.Pod, + container string, + destPath string, + data []byte, +) error { + // umask makes the file unreadable by other users from the moment it is + // created; chmod after the write would leave a window where it is not. + // The byte count is echoed back so a short write is not mistaken for a + // complete one: pg_tde would then authenticate with a truncated token. + var stdout, stderr bytes.Buffer + err := podExec(ctx, pod.Namespace, pod.Name, container, + bytes.NewReader(data), &stdout, &stderr, + "bash", "-ceu", fmt.Sprintf("umask 077; cat > %s; wc -c < %s", destPath, destPath)) + if err != nil { + return errors.Wrapf(err, "write %s: %s", destPath, stderr.String()) + } + + written, err := strconv.Atoi(strings.TrimSpace(stdout.String())) + if err != nil { + return errors.Wrapf(err, "check size of %s", destPath) + } + if written != len(data) { + return errors.Errorf("wrote %d of %d bytes to %s", written, len(data), destPath) + } + + return nil +} + +// removeTempFiles removes files from a pod container. Missing files are not an +// error; anything else is, including a file that could not be unlinked. +func removeTempFiles( + ctx context.Context, + podExec runtime.PodExecutor, + pod *corev1.Pod, + container string, + paths ...string, +) error { + var stdout, stderr bytes.Buffer + err := podExec(ctx, pod.Namespace, pod.Name, container, + nil, &stdout, &stderr, + "bash", "-ceu", fmt.Sprintf("rm -f %s", strings.Join(paths, " "))) + if err != nil { + return errors.Wrapf(err, "remove %s: %s", strings.Join(paths, ", "), stderr.String()) + } + return nil +} + // reconcilePostgresUsers writes the objects necessary to manage users and their // passwords in PostgreSQL. func (r *Reconciler) reconcilePostgresUsers( @@ -774,7 +1187,7 @@ func (r *Reconciler) reconcilePostgresUsersInPostgreSQL( return postgres.WriteUsersInPostgreSQL(ctx, cluster, exec, specUsers, verifiers) } - revision, err := safeHash32(func(hasher io.Writer) error { + revision, err := util.SafeHash32(func(hasher io.Writer) error { // Discard log messages about executing SQL. return write(logging.NewContext(ctx, logging.Discard()), func( _ context.Context, stdin io.Reader, _, _ io.Writer, command ...string, diff --git a/internal/controller/postgrescluster/util.go b/internal/controller/postgrescluster/util.go index 68c692a0f2..d0492da14e 100644 --- a/internal/controller/postgrescluster/util.go +++ b/internal/controller/postgrescluster/util.go @@ -6,14 +6,11 @@ package postgrescluster import ( "fmt" - "hash/fnv" - "io" gover "github.com/hashicorp/go-version" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/rand" "github.com/percona/percona-postgresql-operator/v2/internal/initialize" "github.com/percona/percona-postgresql-operator/v2/internal/naming" @@ -312,15 +309,3 @@ func jobCompleted(job *batchv1.Job) bool { } return false } - -// safeHash32 runs content and returns a short alphanumeric string that -// represents everything written to w. The string is unlikely to have bad words -// and is safe to store in the Kubernetes API. This is the same algorithm used -// by ControllerRevision's "controller.kubernetes.io/hash". -func safeHash32(content func(w io.Writer) error) (string, error) { - hash := fnv.New32() - if err := content(hash); err != nil { - return "", err - } - return rand.SafeEncodeString(fmt.Sprint(hash.Sum32())), nil -} diff --git a/internal/controller/postgrescluster/util_test.go b/internal/controller/postgrescluster/util_test.go index f73a4a49f0..a3f3733c6b 100644 --- a/internal/controller/postgrescluster/util_test.go +++ b/internal/controller/postgrescluster/util_test.go @@ -5,10 +5,8 @@ package postgrescluster import ( - "io" "testing" - "github.com/pkg/errors" "gotest.tools/v3/assert" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -20,27 +18,6 @@ import ( "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" ) -func TestSafeHash32(t *testing.T) { - expected := errors.New("whomp") - - _, err := safeHash32(func(io.Writer) error { return expected }) - assert.Equal(t, err, expected) - - stuff, err := safeHash32(func(w io.Writer) error { - _, _ = w.Write([]byte(`some stuff`)) - return nil - }) - assert.NilError(t, err) - assert.Equal(t, stuff, "574b4c7d87", "expected alphanumeric") - - same, err := safeHash32(func(w io.Writer) error { - _, _ = w.Write([]byte(`some stuff`)) - return nil - }) - assert.NilError(t, err) - assert.Equal(t, same, stuff, "expected deterministic hash") -} - func TestAddDevSHM(t *testing.T) { testCases := []struct { diff --git a/internal/naming/annotations.go b/internal/naming/annotations.go index ec04eb0e9a..9a48074fca 100644 --- a/internal/naming/annotations.go +++ b/internal/naming/annotations.go @@ -81,4 +81,7 @@ const ( // is present, the controller will not update the ConfigMap, allowing users to make custom // modifications that won't be overwritten during reconciliation. OverrideConfigAnnotation = perconaAnnotationPrefix + "override-config" + + // K8SPG-911 + TDEInstalledAnnotation = perconaAnnotationPrefix + "tde-installed" ) diff --git a/internal/naming/names.go b/internal/naming/names.go index 537a875967..c367dbbf93 100644 --- a/internal/naming/names.go +++ b/internal/naming/names.go @@ -134,6 +134,21 @@ const ( ReplicationCACertPath = "replication/ca.crt" ) +const ( + // PGTDEVolume is the name of the pg_tde secret volume and volume mount in a + // PostgreSQL instance Pod + PGTDEVolume = "pg-tde" + + // PGTDEMountPath is the path for mounting the pg_tde secret + PGTDEMountPath = "/pgconf/tde" + + // PGTDEVaultProvider is the name of the Vault provider + PGTDEVaultProvider = "vault-provider" + + // PGTDEGlobalKey is the name of the global key + PGTDEGlobalKey = "global-master-key" +) + const ( // PGBackRestRepoContainerName is the name assigned to the container used to run pgBackRest PGBackRestRepoContainerName = "pgbackrest" diff --git a/internal/patroni/config.go b/internal/patroni/config.go index 64752d08d0..b64af4e775 100644 --- a/internal/patroni/config.go +++ b/internal/patroni/config.go @@ -158,6 +158,14 @@ func clusterYAML( }, } + if cluster.Spec.Extensions.PGTDE.Enabled { + postgresqlSection := root["postgresql"].(map[string]any) + postgresqlSection["bin_name"] = map[string]any{ + "pg_basebackup": "pg_tde_basebackup", + "pg_rewind": "pg_tde_rewind", + } + } + if !ClusterBootstrapped(cluster) { // Patroni has not yet bootstrapped. Populate the "bootstrap.dcs" field to // facilitate it. When Patroni is already bootstrapped, this field is ignored. diff --git a/internal/patroni/config_test.go b/internal/patroni/config_test.go index 8598ca58f6..2bbc645f1c 100644 --- a/internal/patroni/config_test.go +++ b/internal/patroni/config_test.go @@ -154,7 +154,6 @@ watchdog: cluster.Labels = map[string]string{ v1beta1.LabelVersion: "2.9.0", } - cluster.Spec.Patroni = &v1beta1.PatroniSpec{} cluster.Spec.Patroni.Default() data, err := clusterYAML(cluster, postgres.HBAs{}, postgres.Parameters{}) @@ -194,6 +193,55 @@ watchdog: assert.Assert(t, !exists, "expected remove_data_directory_on_diverged_timelines to be absent for version < 2.9.0") }) + t.Run("PGTDE enabled adds bin_name", func(t *testing.T) { + cluster := new(v1beta1.PostgresCluster) + err := cluster.Default(context.Background(), nil) + assert.NilError(t, err) + cluster.Namespace = "some-namespace" + cluster.Name = "cluster-name" + + cluster.Spec.PostgresVersion = 17 + cluster.Spec.Extensions.PGTDE.Enabled = true + + data, err := clusterYAML(cluster, postgres.HBAs{}, postgres.Parameters{}) + assert.NilError(t, err) + + var parsed map[string]any + assert.NilError(t, yaml.Unmarshal([]byte(data), &parsed)) + + pgSection, ok := parsed["postgresql"].(map[string]any) + assert.Assert(t, ok, "expected postgresql section") + binName, ok := pgSection["bin_name"].(map[string]any) + assert.Assert(t, ok, "expected postgresql.bin_name section") + + assert.Equal(t, binName["pg_basebackup"], "pg_tde_basebackup") + assert.Equal(t, binName["pg_rewind"], "pg_tde_rewind") + }) + + t.Run("PGTDE disabled no bin_name", func(t *testing.T) { + cluster := new(v1beta1.PostgresCluster) + err := cluster.Default(context.Background(), nil) + assert.NilError(t, err) + cluster.Namespace = "some-namespace" + cluster.Name = "cluster-name" + cluster.Labels = map[string]string{ + v1beta1.LabelVersion: "2.9.0", + } + cluster.Spec.Patroni = &v1beta1.PatroniSpec{} + cluster.Spec.Patroni.Default() + + data, err := clusterYAML(cluster, postgres.HBAs{}, postgres.Parameters{}) + assert.NilError(t, err) + + var parsed map[string]any + assert.NilError(t, yaml.Unmarshal([]byte(data), &parsed)) + + pgSection, ok := parsed["postgresql"].(map[string]any) + assert.Assert(t, ok, "expected postgresql section") + _, hasBinName := pgSection["bin_name"] + assert.Assert(t, !hasBinName, "expected no bin_name when PGTDE is disabled") + }) + t.Run(">PG10", func(t *testing.T) { cluster := new(v1beta1.PostgresCluster) err := cluster.Default(context.Background(), nil) diff --git a/internal/pgbackrest/config.go b/internal/pgbackrest/config.go index 0ed478dbd3..d8b9059621 100644 --- a/internal/pgbackrest/config.go +++ b/internal/pgbackrest/config.go @@ -173,7 +173,7 @@ func MakePGBackrestLogDir(template *corev1.PodTemplateSpec, // - Renames the data directory as needed to bootstrap the cluster using the restored database. // This ensures compatibility with the "existing" bootstrap method that is included in the // Patroni config when bootstrapping a cluster using an existing data directory. -func RestoreCommand(pgdata, hugePagesSetting, fetchKeyCommand string, _ []*corev1.PersistentVolumeClaim, args ...string) []string { +func RestoreCommand(pgdata, hugePagesSetting, fetchKeyCommand string, _ []*corev1.PersistentVolumeClaim, tdeEnabled bool, args ...string) []string { ps := postgres.NewParameterSet() ps.Add("data_directory", pgdata) ps.Add("huge_pages", hugePagesSetting) @@ -187,6 +187,10 @@ func RestoreCommand(pgdata, hugePagesSetting, fetchKeyCommand string, _ []*corev // progress during recovery. ps.Add("hot_standby", "on") + if tdeEnabled { + ps.Add("shared_preload_libraries", "pg_tde") + } + if fetchKeyCommand != "" { ps.Add("encryption_key_command", fetchKeyCommand) } diff --git a/internal/pgbackrest/config_test.go b/internal/pgbackrest/config_test.go index c931558e21..b6c89d0bb3 100644 --- a/internal/pgbackrest/config_test.go +++ b/internal/pgbackrest/config_test.go @@ -340,7 +340,7 @@ func TestRestoreCommand(t *testing.T) { "--stanza=" + DefaultStanzaName, "--pg1-path=" + pgdata, "--repo=1", } - command := RestoreCommand(pgdata, "try", "", nil, strings.Join(opts, " ")) + command := RestoreCommand(pgdata, "try", "", nil, false, strings.Join(opts, " ")) assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) assert.Assert(t, len(command) > 3) @@ -357,7 +357,7 @@ func TestRestoreCommand(t *testing.T) { func TestRestoreCommandPrettyYAML(t *testing.T) { assert.Assert(t, cmp.MarshalContains( - RestoreCommand("/dir", "try", "", nil, "--options"), + RestoreCommand("/dir", "try", "", nil, false, "--options"), "\n- |", ), "expected literal block scalar") @@ -366,7 +366,7 @@ func TestRestoreCommandPrettyYAML(t *testing.T) { func TestRestoreCommandTDE(t *testing.T) { assert.Assert(t, cmp.MarshalContains( - RestoreCommand("/dir", "try", "echo testValue", nil, "--options"), + RestoreCommand("/dir", "try", "echo testValue", nil, false, "--options"), "encryption_key_command = 'echo testValue'", ), "expected encryption_key_command setting") diff --git a/internal/pgcron/postgres.go b/internal/pgcron/postgres.go index 6c39593ac6..adf3d1e028 100644 --- a/internal/pgcron/postgres.go +++ b/internal/pgcron/postgres.go @@ -43,4 +43,5 @@ func DisableInPostgreSQL(ctx context.Context, exec postgres.Executor) error { func PostgreSQLParameters(outParameters *postgres.Parameters) { outParameters.Mandatory.AppendToList("shared_preload_libraries", "pg_cron") + outParameters.Mandatory.Add("cron.database_name", "postgres") } diff --git a/internal/pgcron/postgres_test.go b/internal/pgcron/postgres_test.go index 73cf202dad..df7b497d92 100644 --- a/internal/pgcron/postgres_test.go +++ b/internal/pgcron/postgres_test.go @@ -69,6 +69,7 @@ func TestPostgreSQLParameters(t *testing.T) { assert.Assert(t, parameters.Default == nil) assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ + "cron.database_name": "postgres", "shared_preload_libraries": "pg_cron", }) @@ -77,6 +78,7 @@ func TestPostgreSQLParameters(t *testing.T) { assert.Assert(t, parameters.Default == nil) assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ + "cron.database_name": "postgres", "shared_preload_libraries": "some,existing,pg_cron", }) } diff --git a/internal/pgtde/postgres.go b/internal/pgtde/postgres.go new file mode 100644 index 0000000000..f7b4a21336 --- /dev/null +++ b/internal/pgtde/postgres.go @@ -0,0 +1,415 @@ +package pgtde + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + + "github.com/percona/percona-postgresql-operator/v2/internal/logging" + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + "github.com/percona/percona-postgresql-operator/v2/internal/util" + crunchyv1beta1 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +const ( + // TempTokenPath is where the new vault token is written inside the pod + // during a vault provider change (before the volume is updated). + // Stored under /pgdata so it survives pod restarts (persistent volume). + TempTokenPath = "/pgdata/tde-new-token" // nolint:gosec + // TempCAPath is where the new CA certificate is written inside the pod + // during a vault provider change (before the volume is updated). + // Stored under /pgdata so it survives pod restarts (persistent volume). + TempCAPath = "/pgdata/tde-new-ca.crt" +) + +// enableInPostgreSQL installs pg_tde extension in every database. +func enableInPostgreSQL(ctx context.Context, exec postgres.Executor) error { + log := logging.FromContext(ctx) + + stdout, stderr, err := exec.ExecInAllDatabases(ctx, + strings.Join([]string{ + `SET client_min_messages = WARNING;`, + `CREATE EXTENSION IF NOT EXISTS pg_tde;`, + `ALTER EXTENSION pg_tde UPDATE;`, + }, "\n"), + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one command fails. + "QUIET": "on", // Do not print successful commands to stdout. + }) + + log.V(1).Info("enabled pg_tde", "stdout", stdout, "stderr", stderr) + + return err +} + +func disableInPostgreSQL(ctx context.Context, exec postgres.Executor) error { + log := logging.FromContext(ctx) + + stdout, stderr, err := exec.ExecInAllDatabases(ctx, + strings.Join([]string{ + `SET client_min_messages = WARNING;`, + `DROP EXTENSION IF EXISTS pg_tde;`, + }, "\n"), + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one command fails. + "QUIET": "on", // Do not print successful commands to stdout. + }) + + log.V(1).Info("disabled pg_tde", "stdout", stdout, "stderr", stderr) + + return err +} + +// ReconcileExtension installs or drops the pg_tde extension according to the spec. +func ReconcileExtension(ctx context.Context, exec postgres.Executor, cluster *crunchyv1beta1.PostgresCluster) error { + if !cluster.Spec.Extensions.PGTDE.Enabled { + return disableInPostgreSQL(ctx, exec) + } + + return enableInPostgreSQL(ctx, exec) +} + +// ReportExtension records the outcome of a ReconcileExtension. +// The PGTDEEnabled condition decides whether pg_tde is in shared_preload_libraries +// and whether instance Pods carry the vault volume. +func ReportExtension(cluster *crunchyv1beta1.PostgresCluster, record record.EventRecorder, err error) { + enabled := cluster.Spec.Extensions.PGTDE.Enabled + + if err != nil { + // Leave the condition alone: a failed DROP means the extension is + // still installed, and a failed CREATE means whatever was there + // before still is. + if enabled { + record.Event(cluster, corev1.EventTypeWarning, + "PGTDEInstallFailed", "Unable to install pg_tde") + } else { + record.Event(cluster, corev1.EventTypeWarning, + "PGTDEDisableFailed", "Unable to disable pg_tde") + } + return + } + + condition := metav1.Condition{ + Type: crunchyv1beta1.PGTDEEnabled, + Status: metav1.ConditionTrue, + Reason: "Enabled", + Message: "pg_tde is enabled in PerconaPGCluster", + ObservedGeneration: cluster.GetGeneration(), + } + if !enabled { + condition.Status = metav1.ConditionFalse + condition.Reason = "Disabled" + condition.Message = "pg_tde is disabled in PerconaPGCluster" + } + meta.SetStatusCondition(&cluster.Status.Conditions, condition) +} + +func PostgreSQLParameters(outParameters *postgres.Parameters) { + outParameters.Mandatory.AppendToList("shared_preload_libraries", "pg_tde") + outParameters.Mandatory.Add("pg_tde.wal_encrypt", "off") +} + +// VaultCredentialPaths returns the standard volume mount paths for the vault +// token and CA certificate based on the vault spec's secret key names. +func VaultCredentialPaths(vault *crunchyv1beta1.PGTDEVaultSpec) (tokenPath, caPath string) { + tokenPath = naming.PGTDEMountPath + "/" + vault.TokenSecret.Key + if vault.HasCA() { + caPath = naming.PGTDEMountPath + "/" + vault.CASecret.Key + } + return tokenPath, caPath +} + +// TempVaultCredentialPaths returns the temporary file paths used during a vault +// provider change, before the pod volume is updated with new credentials. +func TempVaultCredentialPaths(vault *crunchyv1beta1.PGTDEVaultSpec) (tokenPath, caPath string) { + tokenPath = TempTokenPath + if vault.HasCA() { + caPath = TempCAPath + } + return tokenPath, caPath +} + +func addVaultProvider(ctx context.Context, exec postgres.Executor, vault *crunchyv1beta1.PGTDEVaultSpec, tokenPath, caPath string) error { + log := logging.FromContext(ctx) + + stdout, stderr, err := exec.Exec(ctx, + strings.NewReader(strings.Join([]string{ + // Quiet NOTICE messages from IF NOT EXISTS statements. + // - https://www.postgresql.org/docs/current/runtime-config-client.html + `SET client_min_messages = WARNING;`, + `SELECT pg_tde_add_global_key_provider_vault_v2( + :'provider_name', :'vault_host', :'vault_mount_path', :'token_path', NULLIF(:'ca_path', '') + );`, + }, "\n")), + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one statement fails. + "QUIET": "on", // Do not print successful statements to stdout. + "provider_name": naming.PGTDEVaultProvider, + "vault_host": vault.Host, + "vault_mount_path": vault.MountPath, + "token_path": tokenPath, + "ca_path": caPath, + }, nil) + + if err != nil { + log.Info("failed to add pg_tde vault provider", "stdout", stdout, "stderr", stderr) + } else { + log.Info("added pg_tde vault provider", "stdout", stdout, "stderr", stderr) + } + + return err +} + +func createGlobalKey(ctx context.Context, exec postgres.Executor, clusterID types.UID) error { + log := logging.FromContext(ctx) + + globalKey := fmt.Sprintf("%s-%s", naming.PGTDEGlobalKey, clusterID) + + stdout, stderr, err := exec.Exec(ctx, + strings.NewReader(strings.Join([]string{ + // Quiet NOTICE messages from IF NOT EXISTS statements. + // - https://www.postgresql.org/docs/current/runtime-config-client.html + `SET client_min_messages = WARNING;`, + `SELECT pg_tde_create_key_using_global_key_provider(:'global_key', :'provider_name');`, + }, "\n")), + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one statement fails. + "QUIET": "on", // Do not print successful statements to stdout. + "provider_name": naming.PGTDEVaultProvider, + "global_key": globalKey, + }, nil) + + if err != nil { + log.Info("failed to create global key", "globalKey", globalKey, "stdout", stdout, "stderr", stderr) + } else { + log.Info("created global key", "globalKey", globalKey, "stdout", stdout, "stderr", stderr) + } + + return err +} + +func setDefaultKey(ctx context.Context, exec postgres.Executor, clusterID types.UID) error { + log := logging.FromContext(ctx) + + globalKey := fmt.Sprintf("%s-%s", naming.PGTDEGlobalKey, clusterID) + + stdout, stderr, err := exec.Exec(ctx, + strings.NewReader(strings.Join([]string{ + // Quiet NOTICE messages from IF NOT EXISTS statements. + // - https://www.postgresql.org/docs/current/runtime-config-client.html + `SET client_min_messages = WARNING;`, + `SELECT pg_tde_set_default_key_using_global_key_provider(:'global_key', :'provider_name');`, + }, "\n")), + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one statement fails. + "QUIET": "on", // Do not print successful statements to stdout. + "provider_name": naming.PGTDEVaultProvider, + "global_key": globalKey, + }, nil) + + if err != nil { + log.Info("failed to set global key", "globalKey", globalKey, "stdout", stdout, "stderr", stderr) + } else { + log.Info("set global key", "globalKey", globalKey, "stdout", stdout, "stderr", stderr) + } + + return err +} + +func changeVaultProvider(ctx context.Context, exec postgres.Executor, vault *crunchyv1beta1.PGTDEVaultSpec, tokenPath, caPath string) error { + log := logging.FromContext(ctx) + + stdout, stderr, err := exec.Exec(ctx, + strings.NewReader(strings.Join([]string{ + // Quiet NOTICE messages from IF NOT EXISTS statements. + // - https://www.postgresql.org/docs/current/runtime-config-client.html + `SET client_min_messages = WARNING;`, + `SELECT pg_tde_change_global_key_provider_vault_v2( + :'provider_name', :'vault_host', :'vault_mount_path', :'token_path', NULLIF(:'ca_path', '') + );`, + }, "\n")), + map[string]string{ + "ON_ERROR_STOP": "on", // Abort when any one statement fails. + "QUIET": "on", // Do not print successful statements to stdout. + "provider_name": naming.PGTDEVaultProvider, + "vault_host": vault.Host, + "vault_mount_path": vault.MountPath, + "token_path": tokenPath, + "ca_path": caPath, + }, nil) + + if err != nil { + log.Info("failed to change pg_tde vault provider", "stdout", stdout, "stderr", stderr) + } else { + log.Info("changed pg_tde vault provider", "stdout", stdout, "stderr", stderr) + } + + return err +} + +// ReconcileVaultProvider configures or updates the pg_tde vault key provider. +// tokenPath and caPath are the file paths inside the pod where the vault +// credentials can be read. For initial setup these are the standard volume +// mount paths; for provider changes they may be temporary file paths. +// +// The provider and the global key may already exist even on the initial setup +// path: a cluster that is deleted and recreated with its PVCs retained, or one +// where pg_tde was disabled and re-enabled, starts with an empty PGTDERevision +// but a populated pg_tde state. Rather than interpreting the error text to +// recognize those cases, each step recovers from a failure by driving the state +// towards the spec and lets the following step decide whether that worked. +func ReconcileVaultProvider(ctx context.Context, exec postgres.Executor, cluster *crunchyv1beta1.PostgresCluster, tokenPath, caPath string) error { + log := logging.FromContext(ctx) + vault := cluster.Spec.Extensions.PGTDE.Vault + + if cluster.Status.PGTDERevision != "" { + return changeVaultProvider(ctx, exec, vault, tokenPath, caPath) + } + + if addErr := addVaultProvider(ctx, exec, vault, tokenPath, caPath); addErr != nil { + // The provider probably exists already. Its configuration belongs to + // whatever created it, which may be an older incarnation of this + // cluster pointing at a different Vault; overwrite it so it matches + // the spec instead of assuming it already does. + log.V(1).Info("could not add pg_tde vault provider, rewriting the existing one", "error", addErr.Error()) + + if err := changeVaultProvider(ctx, exec, vault, tokenPath, caPath); err != nil { + // Neither statement worked, so the provider is not usable. The + // failure to add it is the more useful of the two to report. + return errors.Wrap(addErr, "add vault provider") + } + } + + // Creating the key fails when it already exists, which is expected for a + // recreated cluster. Setting it as the default is the real test of whether + // the provider and the key are usable, so defer to that result. + createErr := createGlobalKey(ctx, exec, cluster.UID) + + if err := setDefaultKey(ctx, exec, cluster.UID); err != nil { + if createErr != nil { + return errors.Wrap(createErr, "create global key") + } + return errors.Wrap(err, "set default key") + } + + return nil +} + +// Phase is where a cluster stands in the two-phase vault credential change +// described on reconcilePGTDEProviders. +type Phase int + +const ( + // InitialSetup means no key provider has been configured yet, so the + // credentials in the spec are the only ones there have ever been. + InitialSetup Phase = iota + + // Configured means the key provider names the credentials in the spec. + Configured + + // StageCredentials means the spec names credentials the key provider + // has not been pointed at yet. Phase 1 has to copy them onto the data + // volumes and repoint the provider before the Pods may mount them. + StageCredentials + + // Finalize means the key provider names the staged copies on the data + // volumes. Phase 2 repoints it at the mount paths and removes them. + Finalize +) + +// vaultChange is the state of a vault credential change. reconcileInstance +// decides whether to hold the Pods' vault volume from it and +// reconcilePGTDEProviders decides which SQL to run; the two have to agree, +// because releasing the volume in a phase that still expects the old +// credentials mounted is what leaves pg_tde unable to fetch its key. +type vaultChange struct { + Phase Phase + + // Paths inside the Pod. The standard pair are the projected Secret's mount + // paths; the temp pair are the copies staged on the data volume. + TokenPath, CAPath string + TempTokenPath, TempCAPath string + + // Revisions matching each pair of paths, to compare with + // cluster.Status.PGTDERevision. + StandardRevision, TempRevision string +} + +// VaultChangeFor derives the change from the spec and the stored revision. +func VaultChangeFor(cluster *crunchyv1beta1.PostgresCluster) (vaultChange, error) { + var change vaultChange + var err error + + vault := cluster.Spec.Extensions.PGTDE.Vault + change.TokenPath, change.CAPath = VaultCredentialPaths(vault) + change.TempTokenPath, change.TempCAPath = TempVaultCredentialPaths(vault) + + if change.StandardRevision, err = VaultRevision( + vault, change.TokenPath, change.CAPath); err != nil { + return change, err + } + if change.TempRevision, err = VaultRevision( + vault, change.TempTokenPath, change.TempCAPath); err != nil { + return change, err + } + + switch cluster.Status.PGTDERevision { + case "": + change.Phase = InitialSetup + case change.StandardRevision: + change.Phase = Configured + case change.TempRevision: + change.Phase = Finalize + default: + change.Phase = StageCredentials + } + + return change, nil +} + +// VaultRevision computes a hash of the vault configuration and credential +// paths for comparing with cluster.Status.PGTDERevision. +func VaultRevision(vault *crunchyv1beta1.PGTDEVaultSpec, tokenPath, caPath string) (string, error) { + return util.SafeHash32(func(hasher io.Writer) error { + _, err := fmt.Fprintf(hasher, "%q%q%q%q%q%q%q%q", + vault.Host, vault.MountPath, + vault.TokenSecret.Name, vault.TokenSecret.Key, + vault.CASecret.Name, vault.CASecret.Key, + tokenPath, caPath) + return err + }) +} + +// preserveOldTDEVolume replaces the pg-tde volume in the new pod spec with the +// one from the StatefulSet as it exists in the cluster. This prevents pods from +// restarting with new vault credentials before the vault provider change SQL +// has been executed. +func PreserveOldTDEVolume(podSpec *corev1.PodSpec, existing *appsv1.StatefulSet) { + var oldVolume *corev1.Volume + for i := range existing.Spec.Template.Spec.Volumes { + if existing.Spec.Template.Spec.Volumes[i].Name == naming.PGTDEVolume { + oldVolume = &existing.Spec.Template.Spec.Volumes[i] + break + } + } + if oldVolume == nil { + return + } + + for i := range podSpec.Volumes { + if podSpec.Volumes[i].Name == naming.PGTDEVolume { + podSpec.Volumes[i] = *oldVolume + return + } + } +} diff --git a/internal/pgtde/postgres_test.go b/internal/pgtde/postgres_test.go new file mode 100644 index 0000000000..be36fbc178 --- /dev/null +++ b/internal/pgtde/postgres_test.go @@ -0,0 +1,686 @@ +package pgtde + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/pkg/errors" + "gotest.tools/v3/assert" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + crunchyv1beta1 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +func TestEnableInPostgreSQL(t *testing.T) { + expected := errors.New("whoops") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + + assert.Assert(t, strings.Contains(strings.Join(command, "\n"), + `SELECT datname FROM pg_catalog.pg_database`, + ), "expected all databases and templates") + + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + assert.Equal(t, string(b), strings.Join([]string{ + `SET client_min_messages = WARNING;`, + `CREATE EXTENSION IF NOT EXISTS pg_tde;`, + `ALTER EXTENSION pg_tde UPDATE;`, + }, "\n")) + + return expected + } + + ctx := t.Context() + assert.Equal(t, expected, enableInPostgreSQL(ctx, exec)) +} + +func TestDisableInPostgreSQL(t *testing.T) { + expected := errors.New("whoops") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + + assert.Assert(t, strings.Contains(strings.Join(command, "\n"), + `SELECT datname FROM pg_catalog.pg_database`, + ), "expected all databases and templates") + + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + assert.Equal(t, string(b), strings.Join([]string{ + `SET client_min_messages = WARNING;`, + `DROP EXTENSION IF EXISTS pg_tde;`, + }, "\n")) + + return expected + } + + ctx := context.Background() + assert.Equal(t, expected, disableInPostgreSQL(ctx, exec)) +} + +func TestPostgreSQLParameters(t *testing.T) { + parameters := postgres.Parameters{ + Mandatory: postgres.NewParameterSet(), + } + + // No comma when empty. + PostgreSQLParameters(¶meters) + + assert.Assert(t, parameters.Default == nil) + assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ + "shared_preload_libraries": "pg_tde", + "pg_tde.wal_encrypt": "off", + }) + + // Appended when not empty. + parameters.Mandatory.Add("shared_preload_libraries", "some,existing") + PostgreSQLParameters(¶meters) + + assert.Assert(t, parameters.Default == nil) + assert.DeepEqual(t, parameters.Mandatory.AsMap(), map[string]string{ + "shared_preload_libraries": "some,existing,pg_tde", + "pg_tde.wal_encrypt": "off", + }) +} + +func TestAddVaultProvider(t *testing.T) { + t.Run("with CA secret", func(t *testing.T) { + expected := errors.New("whoops") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + sql := string(b) + + assert.Assert(t, strings.Contains(sql, "pg_tde_add_global_key_provider_vault_v2")) + + joined := strings.Join(command, " ") + assert.Assert(t, strings.Contains(joined, "--set=provider_name="+naming.PGTDEVaultProvider)) + assert.Assert(t, strings.Contains(joined, "--set=vault_host=https://vault.example.com")) + assert.Assert(t, strings.Contains(joined, "--set=vault_mount_path=secret/data")) + assert.Assert(t, strings.Contains(joined, "--set=token_path="+naming.PGTDEMountPath+"/token-key")) + assert.Assert(t, strings.Contains(joined, "--set=ca_path="+naming.PGTDEMountPath+"/ca-key")) + + return expected + } + + ctx := context.Background() + vault := &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "token-secret", + Key: "token-key", + }, + CASecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "ca-secret", + Key: "ca-key", + }, + } + tokenPath, caPath := VaultCredentialPaths(vault) + assert.Equal(t, expected, addVaultProvider(ctx, exec, vault, tokenPath, caPath)) + }) + + t.Run("does not interpret stderr", func(t *testing.T) { + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + // psql exits zero, so the statement succeeded no matter what a + // NOTICE or a localized message on stderr happens to say. + _, _ = stderr.Write([]byte("ERROR: already exists")) + return nil + } + + ctx := context.Background() + vault := &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "token-secret", + Key: "token-key", + }, + } + tokenPath, caPath := VaultCredentialPaths(vault) + assert.NilError(t, addVaultProvider(ctx, exec, vault, tokenPath, caPath)) + }) + + t.Run("without CA secret", func(t *testing.T) { + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + joined := strings.Join(command, " ") + assert.Assert(t, strings.Contains(joined, "--set=ca_path="), + "ca_path should be set to empty string") + + return nil + } + + ctx := t.Context() + vault := &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "token-secret", + Key: "token-key", + }, + } + tokenPath, caPath := VaultCredentialPaths(vault) + assert.NilError(t, addVaultProvider(ctx, exec, vault, tokenPath, caPath)) + }) +} + +func TestCreateGlobalKey(t *testing.T) { + t.Run("success", func(t *testing.T) { + expected := errors.New("whoops") + clusterID := types.UID("test-cluster-uid") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + sql := string(b) + + assert.Assert(t, strings.Contains(sql, "pg_tde_create_key_using_global_key_provider")) + + joined := strings.Join(command, " ") + assert.Assert(t, strings.Contains(joined, "--set=provider_name="+naming.PGTDEVaultProvider)) + assert.Assert(t, strings.Contains(joined, + "--set=global_key="+fmt.Sprintf("%s-%s", naming.PGTDEGlobalKey, clusterID))) + + return expected + } + + ctx := t.Context() + assert.Equal(t, expected, createGlobalKey(ctx, exec, clusterID)) + }) + + t.Run("does not interpret stderr", func(t *testing.T) { + clusterID := types.UID("test-cluster-uid") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + // Whether the key already existed is decided by setDefaultKey, + // not by reading the message psql printed. + _, _ = stderr.Write([]byte("ERROR: already exists")) + return nil + } + + ctx := t.Context() + assert.NilError(t, createGlobalKey(ctx, exec, clusterID)) + }) +} + +func TestSetDefaultKey(t *testing.T) { + t.Run("success", func(t *testing.T) { + expected := errors.New("whoops") + clusterID := types.UID("test-cluster-uid") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + sql := string(b) + + assert.Assert(t, strings.Contains(sql, "pg_tde_set_default_key_using_global_key_provider")) + + joined := strings.Join(command, " ") + assert.Assert(t, strings.Contains(joined, "--set=provider_name="+naming.PGTDEVaultProvider)) + assert.Assert(t, strings.Contains(joined, + "--set=global_key="+fmt.Sprintf("%s-%s", naming.PGTDEGlobalKey, clusterID))) + + return expected + } + + ctx := context.Background() + assert.Equal(t, expected, setDefaultKey(ctx, exec, clusterID)) + }) +} + +func TestChangeVaultProvider(t *testing.T) { + t.Run("with CA secret", func(t *testing.T) { + expected := errors.New("whoops") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + sql := string(b) + + assert.Assert(t, strings.Contains(sql, "pg_tde_change_global_key_provider_vault_v2")) + + joined := strings.Join(command, " ") + assert.Assert(t, strings.Contains(joined, "--set=provider_name="+naming.PGTDEVaultProvider)) + assert.Assert(t, strings.Contains(joined, "--set=vault_host=https://vault.example.com")) + assert.Assert(t, strings.Contains(joined, "--set=vault_mount_path=secret/data")) + assert.Assert(t, strings.Contains(joined, "--set=token_path="+naming.PGTDEMountPath+"/token-key")) + assert.Assert(t, strings.Contains(joined, "--set=ca_path="+naming.PGTDEMountPath+"/ca-key")) + + return expected + } + + ctx := context.Background() + vault := &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "token-secret", + Key: "token-key", + }, + CASecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "ca-secret", + Key: "ca-key", + }, + } + tokenPath, caPath := VaultCredentialPaths(vault) + assert.Equal(t, expected, changeVaultProvider(ctx, exec, vault, tokenPath, caPath)) + }) + + t.Run("without CA secret", func(t *testing.T) { + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + joined := strings.Join(command, " ") + assert.Assert(t, strings.Contains(joined, "--set=ca_path="), + "ca_path should be set to empty string") + + return nil + } + + ctx := context.Background() + vault := &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "token-secret", + Key: "token-key", + }, + } + tokenPath, caPath := VaultCredentialPaths(vault) + assert.NilError(t, changeVaultProvider(ctx, exec, vault, tokenPath, caPath)) + }) +} + +func TestReconcileExtension(t *testing.T) { + // ReconcileExtension must stay free of side effects on the cluster: the + // controller runs it against a fake executor to hash the SQL it would send. + t.Run("reports nothing", func(t *testing.T) { + for _, enabled := range []bool{true, false} { + var sql string + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + sql = string(b) + return nil + } + + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = enabled + + assert.NilError(t, ReconcileExtension(t.Context(), exec, cluster)) + assert.Equal(t, len(cluster.Status.Conditions), 0, + "a dry run must not claim pg_tde reached any state") + + if enabled { + assert.Assert(t, strings.Contains(sql, "CREATE EXTENSION IF NOT EXISTS pg_tde")) + } else { + assert.Assert(t, strings.Contains(sql, "DROP EXTENSION IF EXISTS pg_tde")) + } + } + }) + + t.Run("propagates errors", func(t *testing.T) { + expected := errors.New("whoops") + exec := func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + return expected + } + + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = true + + assert.Equal(t, expected, ReconcileExtension(t.Context(), exec, cluster)) + }) +} + +func TestReportExtension(t *testing.T) { + t.Run("enabled successfully", func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = true + cluster.Generation = 2 + + ReportExtension(cluster, recorder, nil) + + condition := meta.FindStatusCondition(cluster.Status.Conditions, crunchyv1beta1.PGTDEEnabled) + assert.Assert(t, condition != nil) + assert.Equal(t, condition.Status, metav1.ConditionTrue) + assert.Equal(t, condition.Reason, "Enabled") + assert.Equal(t, condition.Message, "pg_tde is enabled in PerconaPGCluster") + assert.Equal(t, condition.ObservedGeneration, int64(2)) + }) + + t.Run("disabled successfully", func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = false + cluster.Generation = 1 + + ReportExtension(cluster, recorder, nil) + + condition := meta.FindStatusCondition(cluster.Status.Conditions, crunchyv1beta1.PGTDEEnabled) + assert.Assert(t, condition != nil) + assert.Equal(t, condition.Status, metav1.ConditionFalse) + assert.Equal(t, condition.Reason, "Disabled") + assert.Equal(t, condition.Message, "pg_tde is disabled in PerconaPGCluster") + assert.Equal(t, condition.ObservedGeneration, int64(1)) + }) + + t.Run("install error records event", func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = true + + ReportExtension(cluster, recorder, errors.New("enable failed")) + + select { + case event := <-recorder.Events: + assert.Assert(t, strings.Contains(event, "PGTDEInstallFailed")) + assert.Assert(t, strings.Contains(event, "Unable to install pg_tde")) + default: + t.Fatal("expected event to be recorded") + } + + assert.Equal(t, len(cluster.Status.Conditions), 0, + "a failed CREATE EXTENSION must not report pg_tde as enabled") + }) + + t.Run("disable error records event", func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = false + + ReportExtension(cluster, recorder, errors.New("disable failed")) + + select { + case event := <-recorder.Events: + assert.Assert(t, strings.Contains(event, "PGTDEDisableFailed")) + assert.Assert(t, strings.Contains(event, "Unable to disable pg_tde")) + default: + t.Fatal("expected event to be recorded") + } + }) + + t.Run("failure keeps the previous condition", func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Enabled = false + + // pg_tde is installed; the user asked to disable it and DROP failed. + ReportExtension(cluster, recorder, nil) + cluster.Spec.Extensions.PGTDE.Enabled = true + ReportExtension(cluster, recorder, nil) + cluster.Spec.Extensions.PGTDE.Enabled = false + ReportExtension(cluster, recorder, errors.New("nope")) + + condition := meta.FindStatusCondition(cluster.Status.Conditions, crunchyv1beta1.PGTDEEnabled) + assert.Assert(t, condition != nil) + assert.Equal(t, condition.Status, metav1.ConditionTrue, + "the extension is still installed, so it must stay in shared_preload_libraries") + }) +} + +func TestReconcileVaultProvider(t *testing.T) { + vault := &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{ + Name: "token-secret", + Key: "token-key", + }, + } + tokenPath, caPath := VaultCredentialPaths(vault) + + // statement names the pg_tde function a psql invocation called, so the + // tests below can describe the expected sequence instead of counting. + statement := func(sql string) string { + for _, name := range []string{ + "pg_tde_add_global_key_provider_vault_v2", + "pg_tde_change_global_key_provider_vault_v2", + "pg_tde_create_key_using_global_key_provider", + "pg_tde_set_default_key_using_global_key_provider", + } { + if strings.Contains(sql, name) { + return name + } + } + return sql + } + + // execSequence returns an Executor that records the pg_tde function each + // call ran and fails the calls named in failures. + execSequence := func(called *[]string, failures map[string]error) postgres.Executor { + return func( + _ context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + b, err := io.ReadAll(stdin) + assert.NilError(t, err) + + name := statement(string(b)) + *called = append(*called, name) + return failures[name] + } + } + + newCluster := func(revision string) *crunchyv1beta1.PostgresCluster { + cluster := &crunchyv1beta1.PostgresCluster{} + cluster.Spec.Extensions.PGTDE.Vault = vault + cluster.Status.PGTDERevision = revision + cluster.UID = "test-uid" + return cluster + } + + t.Run("initial setup", func(t *testing.T) { + var called []string + err := ReconcileVaultProvider(t.Context(), execSequence(&called, nil), + newCluster(""), tokenPath, caPath) + + assert.NilError(t, err) + assert.DeepEqual(t, called, []string{ + "pg_tde_add_global_key_provider_vault_v2", + "pg_tde_create_key_using_global_key_provider", + "pg_tde_set_default_key_using_global_key_provider", + }) + }) + + t.Run("existing provider is rewritten", func(t *testing.T) { + // A cluster recreated on retained PVCs: the provider is already there, + // possibly pointing at a different Vault than the spec asks for. + var called []string + err := ReconcileVaultProvider(t.Context(), + execSequence(&called, map[string]error{ + "pg_tde_add_global_key_provider_vault_v2": errors.New("already exists"), + }), + newCluster(""), tokenPath, caPath) + + assert.NilError(t, err) + assert.DeepEqual(t, called, []string{ + "pg_tde_add_global_key_provider_vault_v2", + // The existing provider must be overwritten rather than trusted + // to match the spec. + "pg_tde_change_global_key_provider_vault_v2", + "pg_tde_create_key_using_global_key_provider", + "pg_tde_set_default_key_using_global_key_provider", + }) + }) + + t.Run("provider unusable", func(t *testing.T) { + // Neither statement works, so this is not an "already exists" case. + expectedErr := errors.New("add vault provider: vault is unreachable") + var called []string + err := ReconcileVaultProvider(t.Context(), + execSequence(&called, map[string]error{ + "pg_tde_add_global_key_provider_vault_v2": errors.New("vault is unreachable"), + "pg_tde_change_global_key_provider_vault_v2": errors.New("no such provider"), + }), + newCluster(""), tokenPath, caPath) + + assert.Equal(t, expectedErr.Error(), err.Error()) + assert.DeepEqual(t, called, []string{ + "pg_tde_add_global_key_provider_vault_v2", + "pg_tde_change_global_key_provider_vault_v2", + }) + }) + + t.Run("existing key", func(t *testing.T) { + // Creating the key fails because it is already there; setting it as + // the default proves the state is good. + var called []string + err := ReconcileVaultProvider(t.Context(), + execSequence(&called, map[string]error{ + "pg_tde_create_key_using_global_key_provider": errors.New("already exists"), + }), + newCluster(""), tokenPath, caPath) + + assert.NilError(t, err) + assert.DeepEqual(t, called, []string{ + "pg_tde_add_global_key_provider_vault_v2", + "pg_tde_create_key_using_global_key_provider", + "pg_tde_set_default_key_using_global_key_provider", + }) + }) + + t.Run("key unusable", func(t *testing.T) { + expectedErr := errors.New("create global key: permission denied") + var called []string + err := ReconcileVaultProvider(t.Context(), + execSequence(&called, map[string]error{ + "pg_tde_create_key_using_global_key_provider": errors.New("permission denied"), + "pg_tde_set_default_key_using_global_key_provider": errors.New("key not found"), + }), + newCluster(""), tokenPath, caPath) + + assert.Equal(t, expectedErr.Error(), err.Error()) + }) + + t.Run("set default key fails", func(t *testing.T) { + expectedErr := errors.New("set default key: oops") + var called []string + err := ReconcileVaultProvider(t.Context(), + execSequence(&called, map[string]error{ + "pg_tde_set_default_key_using_global_key_provider": errors.New("oops"), + }), + newCluster(""), tokenPath, caPath) + + assert.Equal(t, expectedErr.Error(), err.Error()) + }) + + t.Run("revision set calls changeVaultProvider", func(t *testing.T) { + var called []string + err := ReconcileVaultProvider(t.Context(), execSequence(&called, nil), + newCluster("some-revision"), tokenPath, caPath) + + assert.NilError(t, err) + assert.DeepEqual(t, called, []string{ + "pg_tde_change_global_key_provider_vault_v2", + }) + }) + + t.Run("revision set changeVaultProvider fails", func(t *testing.T) { + expected := errors.New("change error") + var called []string + err := ReconcileVaultProvider(t.Context(), + execSequence(&called, map[string]error{ + "pg_tde_change_global_key_provider_vault_v2": expected, + }), + newCluster("some-revision"), tokenPath, caPath) + + assert.Equal(t, expected, err, + "an existing cluster must not silently fall back to adding a provider") + }) +} + +// TestVaultCAAgreement pins the three answers that have to match for a CA to +// work: whether it is projected into the Pod, where pg_tde is told to read it +// from, and where it is staged during a provider change. They were once three +// separate expressions over CASecret, and any pair of them disagreeing is a +// configuration the operator cannot serve. +func TestVaultCAAgreement(t *testing.T) { + t.Parallel() + + vaultWith := func(name, key string) *crunchyv1beta1.PGTDEVaultSpec { + return &crunchyv1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + MountPath: "secret/data", + TokenSecret: crunchyv1beta1.PGTDESecretObjectReference{Name: "vault", Key: "token"}, + CASecret: crunchyv1beta1.PGTDESecretObjectReference{Name: name, Key: key}, + } + } + + // projectsCA reports whether the Pod would mount a CA certificate. The + // volume projects the token and nothing else until a CA is configured, and + // the two secrets may share a name, so count the sources rather than try to + // tell them apart by name. + projectsCA := func(vault *crunchyv1beta1.PGTDEVaultSpec) bool { + sources := postgres.PGTDEVolume(vault).Projected.Sources + assert.Assert(t, len(sources) == 1 || len(sources) == 2, + "expected the token and at most a CA, got %v", sources) + return len(sources) == 2 + } + + for _, tc := range []struct { + name string + vault *crunchyv1beta1.PGTDEVaultSpec + expected bool + }{ + {"Both", vaultWith("vault", "ca.crt"), true}, + {"Neither", vaultWith("", ""), false}, + // Half a reference cannot be resolved, so it is no CA at all. The CRD + // requires both once caSecret is given, but nothing in the operator + // should depend on that to stay consistent. + {"NameOnly", vaultWith("vault", ""), false}, + {"KeyOnly", vaultWith("", "ca.crt"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + _, caPath := VaultCredentialPaths(tc.vault) + _, tempCAPath := TempVaultCredentialPaths(tc.vault) + + assert.Equal(t, tc.vault.HasCA(), tc.expected) + assert.Equal(t, caPath != "", tc.expected, + "the provider should name a CA only when one is configured") + assert.Equal(t, tempCAPath != "", tc.expected, + "a CA should be staged only when one is configured") + assert.Equal(t, projectsCA(tc.vault), tc.expected, + "the Pod should mount a CA only when one is configured") + }) + } +} diff --git a/internal/pgvector/postgres.go b/internal/pgvector/postgres.go index ba3122a03e..34b5f4fc21 100644 --- a/internal/pgvector/postgres.go +++ b/internal/pgvector/postgres.go @@ -12,9 +12,8 @@ func EnableInPostgreSQL(ctx context.Context, exec postgres.Executor) error { log := logging.FromContext(ctx) stdout, stderr, err := exec.ExecInAllDatabases(ctx, - // Quiet the NOTICE from IF EXISTS, and install the pgAudit event triggers. + // Quiet the NOTICE from IF EXISTS, and create pgvector extension. // - https://www.postgresql.org/docs/current/runtime-config-client.html - // - https://github.com/pgaudit/pgaudit#settings `SET client_min_messages = WARNING; CREATE EXTENSION IF NOT EXISTS vector; ALTER EXTENSION vector UPDATE;`, map[string]string{ "ON_ERROR_STOP": "on", // Abort when any one command fails. @@ -30,9 +29,8 @@ func DisableInPostgreSQL(ctx context.Context, exec postgres.Executor) error { log := logging.FromContext(ctx) stdout, stderr, err := exec.ExecInAllDatabases(ctx, - // Quiet the NOTICE from IF EXISTS, and install the pgAudit event triggers. + // Quiet the NOTICE from IF EXISTS, and drop pgvector extension. // - https://www.postgresql.org/docs/current/runtime-config-client.html - // - https://github.com/pgaudit/pgaudit#settings `SET client_min_messages = WARNING; DROP EXTENSION IF EXISTS vector;`, map[string]string{ "ON_ERROR_STOP": "on", // Abort when any one command fails. @@ -44,5 +42,5 @@ func DisableInPostgreSQL(ctx context.Context, exec postgres.Executor) error { return err } -// PostgreSQLParameters sets the parameters required by pgAudit. +// PostgreSQLParameters sets the parameters required by pgvector. func PostgreSQLParameters(outParameters *postgres.Parameters) {} diff --git a/internal/postgres/reconcile.go b/internal/postgres/reconcile.go index 052e55af06..293bbfd9bd 100644 --- a/internal/postgres/reconcile.go +++ b/internal/postgres/reconcile.go @@ -55,6 +55,59 @@ func AdditionalConfigVolumeMount() corev1.VolumeMount { } } +// PGTDEVolumeMount returns the name and mount path of the token and certificates for KMS. +func PGTDEVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: naming.PGTDEVolume, + MountPath: naming.PGTDEMountPath, + ReadOnly: true, + } +} + +// PGTDEVolume returns the projected volume for pg_tde Vault secrets (token and optional CA cert). +func PGTDEVolume(vault *v1beta1.PGTDEVaultSpec) corev1.Volume { + volume := corev1.Volume{ + Name: naming.PGTDEVolume, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: new(int32(0o600)), + Sources: []corev1.VolumeProjection{ + {Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: vault.TokenSecret.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: vault.TokenSecret.Key, + Path: vault.TokenSecret.Key, + }, + }, + }}, + }, + }, + }, + } + + if vault.HasCA() { + volume.Projected.Sources = append( + volume.Projected.Sources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: vault.CASecret.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: vault.CASecret.Key, + Path: vault.CASecret.Key, + }, + }, + }, + }) + } + + return volume +} + // InstancePod initializes outInstancePod with the database container and the // volumes needed by PostgreSQL. func InstancePod(ctx context.Context, @@ -158,6 +211,11 @@ func InstancePod(ctx context.Context, downwardAPIVolumeMount, } + pgTDEVolumeMount := PGTDEVolumeMount() + if inCluster.Spec.Extensions.PGTDE.Vault != nil { + dbContainerMounts = append(dbContainerMounts, pgTDEVolumeMount) + } + if HugePages2MiRequested(inCluster) { dbContainerMounts = append(dbContainerMounts, corev1.VolumeMount{ @@ -236,6 +294,9 @@ func InstancePod(ctx context.Context, dataVolume, downwardAPIVolume, } + if vault := inCluster.Spec.Extensions.PGTDE.Vault; vault != nil { + outInstancePod.Volumes = append(outInstancePod.Volumes, PGTDEVolume(vault)) + } if HugePages2MiRequested(inCluster) { outInstancePod.Volumes = append(outInstancePod.Volumes, corev1.Volume{ diff --git a/internal/util/hash.go b/internal/util/hash.go new file mode 100644 index 0000000000..df8ce8b4de --- /dev/null +++ b/internal/util/hash.go @@ -0,0 +1,21 @@ +package util + +import ( + "fmt" + "hash/fnv" + "io" + + "k8s.io/apimachinery/pkg/util/rand" +) + +// safeHash32 runs content and returns a short alphanumeric string that +// represents everything written to w. The string is unlikely to have bad words +// and is safe to store in the Kubernetes API. This is the same algorithm used +// by ControllerRevision's "controller.kubernetes.io/hash". +func SafeHash32(content func(w io.Writer) error) (string, error) { + hash := fnv.New32() + if err := content(hash); err != nil { + return "", err + } + return rand.SafeEncodeString(fmt.Sprint(hash.Sum32())), nil +} diff --git a/internal/util/hash_test.go b/internal/util/hash_test.go new file mode 100644 index 0000000000..512baacce9 --- /dev/null +++ b/internal/util/hash_test.go @@ -0,0 +1,30 @@ +package util + +import ( + "io" + "testing" + + "github.com/pkg/errors" + "gotest.tools/v3/assert" +) + +func TestSafeHash32(t *testing.T) { + expected := errors.New("whomp") + + _, err := SafeHash32(func(io.Writer) error { return expected }) + assert.Equal(t, err, expected) + + stuff, err := SafeHash32(func(w io.Writer) error { + _, _ = w.Write([]byte(`some stuff`)) + return nil + }) + assert.NilError(t, err) + assert.Equal(t, stuff, "574b4c7d87", "expected alphanumeric") + + same, err := SafeHash32(func(w io.Writer) error { + _, _ = w.Write([]byte(`some stuff`)) + return nil + }) + assert.NilError(t, err) + assert.Equal(t, same, stuff, "expected deterministic hash") +} diff --git a/percona/controller/pgbackup/controller.go b/percona/controller/pgbackup/controller.go index 061f8d84e9..f8568906f5 100644 --- a/percona/controller/pgbackup/controller.go +++ b/percona/controller/pgbackup/controller.go @@ -682,6 +682,7 @@ func startBackup(ctx context.Context, c client.Client, pb *v2.PerconaPGBackup) e if a := pg.Annotations[pNaming.AnnotationBackupInProgress]; a != "" && a != pb.Name { return errors.Errorf("backup %s already in progress", a) } + if pg.Annotations == nil { pg.Annotations = make(map[string]string) } diff --git a/percona/controller/pgbackup/controller_test.go b/percona/controller/pgbackup/controller_test.go index f12420a197..ec5fd4cfda 100644 --- a/percona/controller/pgbackup/controller_test.go +++ b/percona/controller/pgbackup/controller_test.go @@ -627,6 +627,87 @@ func TestReleaseLeaseIfNeeded(t *testing.T) { }) } +func TestStartBackup(t *testing.T) { + ctx := t.Context() + ns := "test-ns" + clusterName := "my-cluster" + + s := scheme.Scheme + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, v2.AddToScheme(s)) + + newCluster := func() *v2.PerconaPGCluster { + return &v2.PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: ns}, + Spec: v2.PerconaPGClusterSpec{ + CRVersion: "2.9.0", + PostgresVersion: 17, + }, + } + } + + newBackup := func() *v2.PerconaPGBackup { + repo := "repo1" + return &v2.PerconaPGBackup{ + ObjectMeta: metav1.ObjectMeta{Name: "backup-1", Namespace: ns}, + Spec: v2.PerconaPGBackupSpec{ + PGCluster: clusterName, + RepoName: &repo, + }, + } + } + + t.Run("marks the cluster for backup", func(t *testing.T) { + cluster, backup := newCluster(), newBackup() + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(cluster, backup).Build() + + require.NoError(t, startBackup(ctx, cl, backup)) + + updated := &v2.PerconaPGCluster{} + require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(cluster), updated)) + + assert.Equal(t, backup.Name, updated.Annotations[naming.PGBackRestBackup]) + assert.Equal(t, backup.Name, updated.Annotations[pNaming.AnnotationBackupInProgress]) + require.NotNil(t, updated.Spec.Backups.PGBackRest.Manual) + assert.Equal(t, "repo1", updated.Spec.Backups.PGBackRest.Manual.RepoName) + }) + + t.Run("does not persist defaults", func(t *testing.T) { + cluster, backup := newCluster(), newBackup() + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(cluster, backup).Build() + + defaulted := cluster.DeepCopy() + defaulted.Default() + require.NotNil(t, defaulted.Spec.Extensions.BuiltIn.PGStatMonitor) // nolint:staticcheck + require.NotNil(t, defaulted.Spec.Extensions.PGStatMonitor.Enabled) + require.NotNil(t, defaulted.Spec.AutoCreateUserSchema) + + require.NoError(t, startBackup(ctx, cl, backup)) + + updated := &v2.PerconaPGCluster{} + require.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(cluster), updated)) + + assert.Nil(t, updated.Spec.Extensions.BuiltIn.PGStatMonitor, // nolint:staticcheck + "a backup must not write the deprecated builtin extension fields") + assert.Nil(t, updated.Spec.Extensions.PGStatMonitor.Enabled, + "a backup must not decide which extensions the user enabled") + assert.Nil(t, updated.Spec.AutoCreateUserSchema, + "a backup must not fill in unrelated spec defaults") + }) + + t.Run("refuses when another backup is running", func(t *testing.T) { + cluster, backup := newCluster(), newBackup() + cluster.Annotations = map[string]string{ + pNaming.AnnotationBackupInProgress: "other-backup", + } + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(cluster, backup).Build() + + err := startBackup(ctx, cl, backup) + require.Error(t, err) + assert.Contains(t, err.Error(), "other-backup") + }) +} + func TestFindBackupInPGBackrestInfo(t *testing.T) { tests := []struct { name string diff --git a/percona/controller/pgcluster/controller_test.go b/percona/controller/pgcluster/controller_test.go index 19f4b17aad..18289cc8c2 100644 --- a/percona/controller/pgcluster/controller_test.go +++ b/percona/controller/pgcluster/controller_test.go @@ -2399,6 +2399,170 @@ var _ = Describe("CR Validations", Ordered, func() { }) }) }) + + Context("pg_tde validations", Ordered, func() { + When("creating a CR with valid pg_tde configurations", func() { + It("should accept pg_tde enabled with vault on PG 17", func() { + cr, err := readDefaultCR("cr-validation-tde-1", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 17 + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + Vault: &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + TokenSecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-token", + Key: "token", + }, + }, + } + + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + + It("should accept pg_tde disabled without vault", func() { + cr, err := readDefaultCR("cr-validation-tde-2", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 17 + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: false, + } + + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + + It("should accept pg_tde not specified at all", func() { + cr, err := readDefaultCR("cr-validation-tde-3", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 16 + + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + + It("should accept pg_tde disabled with vault on PG < 17", func() { + cr, err := readDefaultCR("cr-validation-tde-4", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 16 + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: false, + Vault: &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + TokenSecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-token", + Key: "token", + }, + }, + } + + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + }) + + When("creating a CR with invalid pg_tde configurations", func() { + It("should reject pg_tde enabled on PG < 17", func() { + cr, err := readDefaultCR("cr-validation-tde-5", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 16 + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + Vault: &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + TokenSecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-token", + Key: "token", + }, + }, + } + + err = k8sClient.Create(ctx, cr) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring( + "pg_tde is only supported for PG17 and above", + )) + }) + + It("should reject pg_tde enabled without vault", func() { + cr, err := readDefaultCR("cr-validation-tde-6", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 17 + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + } + + err = k8sClient.Create(ctx, cr) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring( + "vault is required for enabling pg_tde", + )) + }) + }) + + When("updating a CR with pg_tde transition rules", func() { + It("should reject removing vault while pg_tde is still enabled", func() { + cr, err := readDefaultCR("cr-validation-tde-8", ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = 17 + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + Vault: &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + TokenSecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-token", + Key: "token", + }, + }, + } + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + + updated := cr.DeepCopy() + updated.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: true, + } + + err = k8sClient.Update(ctx, updated) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring( + "vault is required for enabling pg_tde", + )) + }) + + It("should accept disabling pg_tde while keeping vault", func() { + cr := &v2.PerconaPGCluster{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "cr-validation-tde-8", Namespace: ns}, cr)).Should(Succeed()) + + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: false, + Vault: &v1beta1.PGTDEVaultSpec{ + Host: "https://vault.example.com:8200", + TokenSecret: v1beta1.PGTDESecretObjectReference{ + Name: "vault-token", + Key: "token", + }, + }, + } + + Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) + }) + + It("should accept removing vault after pg_tde is disabled", func() { + cr := &v2.PerconaPGCluster{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "cr-validation-tde-8", Namespace: ns}, cr)).Should(Succeed()) + + cr.Spec.Extensions.PGTDE = v1beta1.PGTDESpec{ + Enabled: false, + } + + Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) + }) + }) + }) }) var _ = Describe("Init Container", Ordered, func() { diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 41f32bb54c..e3bd983dff 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -51,6 +51,7 @@ type PerconaPGCluster struct { Status PerconaPGClusterStatus `json:"status,omitempty"` } +// +kubebuilder:validation:XValidation:rule="!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17",message="pg_tde is only supported for PG17 and above" // +kubebuilder:validation:XValidation:rule="!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)",message="PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true" type PerconaPGClusterSpec struct { // +optional @@ -274,58 +275,97 @@ func (cr *PerconaPGCluster) Default() { } } - if cr.Spec.Extensions.BuiltIn.PGStatMonitor == nil { - cr.Spec.Extensions.BuiltIn.PGStatMonitor = new(true) + cr.SetExtensionDefaults() + + if cr.CompareVersion("3.1.0") >= 0 && cr.Spec.Backups.Enabled == nil { + cr.Spec.Backups.Enabled = new(true) + } + + if cr.CompareVersion("2.9.0") < 0 && cr.Spec.Config == nil { + cr.Spec.Config = &crunchyv1beta1.PostgresConfigSpec{} + } + + if cr.Spec.Backups.IsVolumeSnapshotsEnabled() && + cr.Spec.Backups.VolumeSnapshots.Mode == VolumeSnapshotModeOffline && + cr.Spec.Backups.VolumeSnapshots.OfflineConfig == nil { + cr.Spec.Backups.VolumeSnapshots.OfflineConfig = DefaultOfflineSnapshotConfig() + } + + if cr.CompareVersion("2.6.0") >= 0 && cr.Spec.AutoCreateUserSchema == nil { + cr.Spec.AutoCreateUserSchema = new(true) + } +} + +func (cr *PerconaPGCluster) SetExtensionDefaults() { + // for backward compatibility, delete after 3.4.0 + if cr.Spec.Extensions.BuiltIn.PGStatMonitor != nil { + cr.Spec.Extensions.PGStatMonitor.Enabled = cr.Spec.Extensions.BuiltIn.PGStatMonitor + } + if cr.Spec.Extensions.BuiltIn.PGStatStatements != nil { + cr.Spec.Extensions.PGStatStatements.Enabled = cr.Spec.Extensions.BuiltIn.PGStatStatements + } + if cr.Spec.Extensions.BuiltIn.PGAudit != nil { + cr.Spec.Extensions.PGAudit.Enabled = cr.Spec.Extensions.BuiltIn.PGAudit + } + if cr.Spec.Extensions.BuiltIn.PGRepack != nil { + cr.Spec.Extensions.PGRepack.Enabled = cr.Spec.Extensions.BuiltIn.PGRepack + } + if cr.Spec.Extensions.BuiltIn.PGVector != nil { + cr.Spec.Extensions.PGVector.Enabled = cr.Spec.Extensions.BuiltIn.PGVector + } + + if cr.Spec.Extensions.PGStatMonitor.Enabled == nil { + cr.Spec.Extensions.PGStatMonitor.Enabled = new(true) if cr.CompareVersion("2.9.0") >= 0 { var qs PMMQuerySource if cr.PMMEnabled() { qs = cr.Spec.PMM.QuerySource } - cr.Spec.Extensions.BuiltIn.PGStatMonitor = new(qs == PgStatMonitor) + cr.Spec.Extensions.PGStatMonitor.Enabled = new(qs == PgStatMonitor) } } - if cr.Spec.Extensions.BuiltIn.PGStatStatements == nil { - cr.Spec.Extensions.BuiltIn.PGStatStatements = new(false) + if cr.Spec.Extensions.PGStatStatements.Enabled == nil { + cr.Spec.Extensions.PGStatStatements.Enabled = new(false) if cr.CompareVersion("2.9.0") >= 0 { var qs PMMQuerySource if cr.PMMEnabled() { qs = cr.Spec.PMM.QuerySource } - cr.Spec.Extensions.BuiltIn.PGStatStatements = new(qs == PgStatStatements) + cr.Spec.Extensions.PGStatStatements.Enabled = new(qs == PgStatStatements) } } - if cr.Spec.Extensions.BuiltIn.PGAudit == nil { - cr.Spec.Extensions.BuiltIn.PGAudit = new(true) + + if cr.Spec.Extensions.PGAudit.Enabled == nil { + cr.Spec.Extensions.PGAudit.Enabled = new(true) } - if cr.Spec.Extensions.BuiltIn.PGVector == nil { - cr.Spec.Extensions.BuiltIn.PGVector = new(false) + if cr.Spec.Extensions.PGVector.Enabled == nil { + cr.Spec.Extensions.PGVector.Enabled = new(false) } - if cr.Spec.Extensions.BuiltIn.PGRepack == nil { - cr.Spec.Extensions.BuiltIn.PGRepack = new(false) + if cr.Spec.Extensions.PGRepack.Enabled == nil { + cr.Spec.Extensions.PGRepack.Enabled = new(false) } - if cr.Spec.Extensions.BuiltIn.PGCron == nil { - cr.Spec.Extensions.BuiltIn.PGCron = new(false) + if cr.Spec.Extensions.SetUser.Enabled == nil { + cr.Spec.Extensions.SetUser.Enabled = new(false) } - if cr.Spec.Extensions.BuiltIn.SetUser == nil { - cr.Spec.Extensions.BuiltIn.SetUser = new(false) + if cr.Spec.Extensions.PGCron.Enabled == nil { + cr.Spec.Extensions.PGCron.Enabled = new(false) } - if cr.CompareVersion("2.6.0") >= 0 && cr.Spec.AutoCreateUserSchema == nil { - cr.Spec.AutoCreateUserSchema = new(true) + // for backward compatibility, delete after 3.4.0 + if cr.Spec.Extensions.BuiltIn.PGStatMonitor == nil { + cr.Spec.Extensions.BuiltIn.PGStatMonitor = cr.Spec.Extensions.PGStatMonitor.Enabled } - - if cr.CompareVersion("3.1.0") >= 0 && cr.Spec.Backups.Enabled == nil { - cr.Spec.Backups.Enabled = new(true) + if cr.Spec.Extensions.BuiltIn.PGStatStatements == nil { + cr.Spec.Extensions.BuiltIn.PGStatStatements = cr.Spec.Extensions.PGStatStatements.Enabled } - - if cr.CompareVersion("2.9.0") < 0 && cr.Spec.Config == nil { - cr.Spec.Config = &crunchyv1beta1.PostgresConfigSpec{} + if cr.Spec.Extensions.BuiltIn.PGAudit == nil { + cr.Spec.Extensions.BuiltIn.PGAudit = cr.Spec.Extensions.PGAudit.Enabled } - - if cr.Spec.Backups.IsVolumeSnapshotsEnabled() && - cr.Spec.Backups.VolumeSnapshots.Mode == VolumeSnapshotModeOffline && - cr.Spec.Backups.VolumeSnapshots.OfflineConfig == nil { - cr.Spec.Backups.VolumeSnapshots.OfflineConfig = DefaultOfflineSnapshotConfig() + if cr.Spec.Extensions.BuiltIn.PGVector == nil { + cr.Spec.Extensions.BuiltIn.PGVector = cr.Spec.Extensions.PGVector.Enabled + } + if cr.Spec.Extensions.BuiltIn.PGRepack == nil { + cr.Spec.Extensions.BuiltIn.PGRepack = cr.Spec.Extensions.PGRepack.Enabled } } @@ -488,26 +528,27 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun postgresCluster.Spec.InstanceSets = cr.Spec.InstanceSets.ToCrunchy() postgresCluster.Spec.Proxy = cr.Spec.Proxy.ToCrunchy(cr.Spec.CRVersion) - if cr.Spec.Extensions.BuiltIn.PGStatMonitor != nil { - postgresCluster.Spec.Extensions.PGStatMonitor = *cr.Spec.Extensions.BuiltIn.PGStatMonitor + postgresCluster.Spec.Extensions.PGTDE = cr.Spec.Extensions.PGTDE + if cr.Spec.Extensions.PGStatMonitor.Enabled != nil { + postgresCluster.Spec.Extensions.PGStatMonitor = *cr.Spec.Extensions.PGStatMonitor.Enabled } - if cr.Spec.Extensions.BuiltIn.PGStatStatements != nil { - postgresCluster.Spec.Extensions.PGStatStatements = *cr.Spec.Extensions.BuiltIn.PGStatStatements + if cr.Spec.Extensions.PGStatStatements.Enabled != nil { + postgresCluster.Spec.Extensions.PGStatStatements = *cr.Spec.Extensions.PGStatStatements.Enabled } - if cr.Spec.Extensions.BuiltIn.PGAudit != nil { - postgresCluster.Spec.Extensions.PGAudit = *cr.Spec.Extensions.BuiltIn.PGAudit + if cr.Spec.Extensions.PGAudit.Enabled != nil { + postgresCluster.Spec.Extensions.PGAudit = *cr.Spec.Extensions.PGAudit.Enabled } - if cr.Spec.Extensions.BuiltIn.PGVector != nil { - postgresCluster.Spec.Extensions.PGVector = *cr.Spec.Extensions.BuiltIn.PGVector + if cr.Spec.Extensions.PGVector.Enabled != nil { + postgresCluster.Spec.Extensions.PGVector = *cr.Spec.Extensions.PGVector.Enabled } - if cr.Spec.Extensions.BuiltIn.PGRepack != nil { - postgresCluster.Spec.Extensions.PGRepack = *cr.Spec.Extensions.BuiltIn.PGRepack + if cr.Spec.Extensions.PGRepack.Enabled != nil { + postgresCluster.Spec.Extensions.PGRepack = *cr.Spec.Extensions.PGRepack.Enabled } - if cr.Spec.Extensions.BuiltIn.PGCron != nil { - postgresCluster.Spec.Extensions.PGCron = *cr.Spec.Extensions.BuiltIn.PGCron + if cr.Spec.Extensions.PGCron.Enabled != nil { + postgresCluster.Spec.Extensions.PGCron = *cr.Spec.Extensions.PGCron.Enabled } - if cr.Spec.Extensions.BuiltIn.SetUser != nil { - postgresCluster.Spec.Extensions.SetUser = *cr.Spec.Extensions.BuiltIn.SetUser + if cr.Spec.Extensions.SetUser.Enabled != nil { + postgresCluster.Spec.Extensions.SetUser = *cr.Spec.Extensions.SetUser.Enabled } postgresCluster.Spec.TLSOnly = cr.Spec.TLSOnly @@ -918,16 +959,31 @@ type BuiltInExtensionsSpec struct { PGAudit *bool `json:"pg_audit,omitempty"` PGVector *bool `json:"pgvector,omitempty"` PGRepack *bool `json:"pg_repack,omitempty"` - PGCron *bool `json:"pg_cron,omitempty"` - SetUser *bool `json:"set_user,omitempty"` } +type BuiltInExtensionSpec struct { + Enabled *bool `json:"enabled,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)",message="to disable pg_tde first set enabled=false without removing vault and wait for pod restarts" type ExtensionsSpec struct { Image string `json:"image,omitempty"` ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` Storage CustomExtensionsStorageSpec `json:"storage,omitempty"` - BuiltIn BuiltInExtensionsSpec `json:"builtin,omitempty"` - Custom []CustomExtensionSpec `json:"custom,omitempty"` + + // Deprecated: Use extensions. instead. This field will be removed after 3.4.0. + BuiltIn BuiltInExtensionsSpec `json:"builtin,omitempty"` + + PGStatMonitor BuiltInExtensionSpec `json:"pg_stat_monitor,omitempty"` + PGStatStatements BuiltInExtensionSpec `json:"pg_stat_statements,omitempty"` + PGAudit BuiltInExtensionSpec `json:"pg_audit,omitempty"` + PGVector BuiltInExtensionSpec `json:"pgvector,omitempty"` + PGRepack BuiltInExtensionSpec `json:"pg_repack,omitempty"` + PGCron BuiltInExtensionSpec `json:"pg_cron,omitempty"` + SetUser BuiltInExtensionSpec `json:"set_user,omitempty"` + PGTDE crunchyv1beta1.PGTDESpec `json:"pg_tde,omitempty"` + + Custom []CustomExtensionSpec `json:"custom,omitempty"` } type SecretsSpec struct { diff --git a/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go b/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go index 4bbcd26560..65050b714c 100644 --- a/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go +++ b/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go @@ -47,6 +47,26 @@ func (in *Backups) DeepCopy() *Backups { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuiltInExtensionSpec) DeepCopyInto(out *BuiltInExtensionSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuiltInExtensionSpec. +func (in *BuiltInExtensionSpec) DeepCopy() *BuiltInExtensionSpec { + if in == nil { + return nil + } + out := new(BuiltInExtensionSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BuiltInExtensionsSpec) DeepCopyInto(out *BuiltInExtensionsSpec) { *out = *in @@ -75,16 +95,6 @@ func (in *BuiltInExtensionsSpec) DeepCopyInto(out *BuiltInExtensionsSpec) { *out = new(bool) **out = **in } - if in.PGCron != nil { - in, out := &in.PGCron, &out.PGCron - *out = new(bool) - **out = **in - } - if in.SetUser != nil { - in, out := &in.SetUser, &out.SetUser - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuiltInExtensionsSpec. @@ -191,6 +201,14 @@ func (in *ExtensionsSpec) DeepCopyInto(out *ExtensionsSpec) { *out = *in in.Storage.DeepCopyInto(&out.Storage) in.BuiltIn.DeepCopyInto(&out.BuiltIn) + in.PGStatMonitor.DeepCopyInto(&out.PGStatMonitor) + in.PGStatStatements.DeepCopyInto(&out.PGStatStatements) + in.PGAudit.DeepCopyInto(&out.PGAudit) + in.PGVector.DeepCopyInto(&out.PGVector) + in.PGRepack.DeepCopyInto(&out.PGRepack) + in.PGCron.DeepCopyInto(&out.PGCron) + in.SetUser.DeepCopyInto(&out.SetUser) + in.PGTDE.DeepCopyInto(&out.PGTDE) if in.Custom != nil { in, out := &in.Custom, &out.Custom *out = make([]CustomExtensionSpec, len(*in)) diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_test.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_test.go index 4dacccf192..becd075b8b 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_test.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_test.go @@ -44,7 +44,8 @@ metadata: {} spec: backups: pgbackrest: {} - extensions: {} + extensions: + pg_tde: {} instances: null patroni: leaderLeaseDurationSeconds: 30 @@ -76,7 +77,8 @@ metadata: {} spec: backups: pgbackrest: {} - extensions: {} + extensions: + pg_tde: {} instances: - dataVolumeClaimSpec: resources: {} diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go index 37fe1e620b..55c74fefa6 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go @@ -235,6 +235,44 @@ type InitContainerSpec struct { ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` } +type PGTDESecretObjectReference struct { + // +kubebuilder:validation:Required + Name string `json:"name"` + // +kubebuilder:validation:Required + Key string `json:"key"` +} + +type PGTDEVaultSpec struct { + // Host of Vault server. + Host string `json:"host"` + // Name of the secret that contains the access token with read and write access to the mount path. + TokenSecret PGTDESecretObjectReference `json:"tokenSecret"` + // Name of the secret that contains the CA certificate for SSL verification. + CASecret PGTDESecretObjectReference `json:"caSecret,omitempty"` + // The mount point on the Vault server where the key provider should store the keys. + // +kubebuilder:default=secret/data + MountPath string `json:"mountPath,omitempty"` +} + +// HasCA reports whether a CA certificate is configured for verifying the Vault +// server. Both halves of the reference are needed to reach the certificate, so +// this is the single answer for whether to project it into the Pod, where to +// read it from, and whether it belongs in the key provider configuration. +// Those three have to agree: naming a path pg_tde cannot read fails every +// request to Vault, and projecting a key that is not set is rejected by the +// API server. +func (s *PGTDEVaultSpec) HasCA() bool { + return s.CASecret.Name != "" && s.CASecret.Key != "" +} + +// +kubebuilder:validation:XValidation:rule="!has(self.enabled) || (has(self.enabled) && self.enabled == false) || has(self.vault)",message="vault is required for enabling pg_tde" +type PGTDESpec struct { + Enabled bool `json:"enabled,omitempty"` + + Vault *PGTDEVaultSpec `json:"vault,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.pg_tde) || !has(oldSelf.pg_tde.vault) || !has(oldSelf.pg_tde.enabled) || !oldSelf.pg_tde.enabled || has(self.pg_tde.vault)",message="to disable pg_tde first set enabled=false without removing vault and wait for pod restarts" type ExtensionsSpec struct { PGStatMonitor bool `json:"pgStatMonitor,omitempty"` PGAudit bool `json:"pgAudit,omitempty"` @@ -243,6 +281,8 @@ type ExtensionsSpec struct { PGRepack bool `json:"pgRepack,omitempty"` PGCron bool `json:"pgCron,omitempty"` SetUser bool `json:"setUser,omitempty"` + + PGTDE PGTDESpec `json:"pg_tde,omitempty"` } type TLSSpec struct { @@ -430,6 +470,9 @@ type PostgresClusterStatus struct { // Identifies the databases that have been installed into PostgreSQL. DatabaseRevision string `json:"databaseRevision,omitempty"` + // Identifies the pg_tde configuration that have been installed into PostgreSQL. + PGTDERevision string `json:"pgTDERevision,omitempty"` + // Current state of PostgreSQL instances. // +listType=map // +listMapKey=name @@ -503,6 +546,14 @@ const ( PostgresClusterProgressing = "Progressing" ProxyAvailable = "ProxyAvailable" Registered = "Registered" + PGTDEEnabled = "PGTDEEnabled" + + // PGTDEVaultProviderReady reports whether the pg_tde vault key provider + // matches the configuration in the spec. It is false while a credential + // change is in progress and when one has failed. Unlike PGTDEEnabled it + // does not influence shared_preload_libraries or Pod contents; it exists + // so a stalled credential change is visible to the user. + PGTDEVaultProviderReady = "PGTDEVaultProviderReady" ) type PostgresInstanceSetSpec struct { diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go index 19c0f207db..28ea0e4b22 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go @@ -436,6 +436,7 @@ func (in *ExporterSpec) DeepCopy() *ExporterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExtensionsSpec) DeepCopyInto(out *ExtensionsSpec) { *out = *in + in.PGTDE.DeepCopyInto(&out.PGTDE) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtensionsSpec. @@ -1502,6 +1503,58 @@ func (in *PGMonitorSpec) DeepCopy() *PGMonitorSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PGTDESecretObjectReference) DeepCopyInto(out *PGTDESecretObjectReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PGTDESecretObjectReference. +func (in *PGTDESecretObjectReference) DeepCopy() *PGTDESecretObjectReference { + if in == nil { + return nil + } + out := new(PGTDESecretObjectReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PGTDESpec) DeepCopyInto(out *PGTDESpec) { + *out = *in + if in.Vault != nil { + in, out := &in.Vault, &out.Vault + *out = new(PGTDEVaultSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PGTDESpec. +func (in *PGTDESpec) DeepCopy() *PGTDESpec { + if in == nil { + return nil + } + out := new(PGTDESpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PGTDEVaultSpec) DeepCopyInto(out *PGTDEVaultSpec) { + *out = *in + out.TokenSecret = in.TokenSecret + out.CASecret = in.CASecret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PGTDEVaultSpec. +func (in *PGTDEVaultSpec) DeepCopy() *PGTDEVaultSpec { + if in == nil { + return nil + } + out := new(PGTDEVaultSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PGUpgrade) DeepCopyInto(out *PGUpgrade) { *out = *in @@ -2013,7 +2066,7 @@ func (in *PostgresClusterSpec) DeepCopyInto(out *PostgresClusterSpec) { *out = new(PostgresClusterAuthentication) (*in).DeepCopyInto(*out) } - out.Extensions = in.Extensions + in.Extensions.DeepCopyInto(&out.Extensions) if in.InitContainer != nil { in, out := &in.InitContainer, &out.InitContainer *out = new(InitContainerSpec)