From 8100be690187d399583d8eafd68836c0c455b579 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Wed, 15 Jul 2026 17:41:13 +0200 Subject: [PATCH 01/13] Refactor perfomrnace tests, add new grafana dashboards, metrics and pyroscope Signed-off-by: Rendre Greyling --- .env.template | 1 + .gitignore | 2 + .../metrics-resources/alloy-config.alloy | 640 +++++ .../grafana-database-dashboard.json | 407 +++ .../grafana-perf-test-dashboard.json | 2358 +++++++++++++++++ .../grafana-porch-server-dashboard.json | 500 ++++ .../grafana-pyroscope-dashboard.json | 114 + .../grafana-resource-usage-dashboard.json | 1196 +++++++++ .../metrics-resources/prometheus-config.yaml | 43 +- deployments/metrics/grafana-deployment.yaml | 6 + .../metrics/postgres-exporter-deployment.yaml | 98 + deployments/metrics/pyroscope-deployment.yaml | 144 + deployments/porch/2-function-runner.yaml | 20 + .../porch/3-porch-postgres-bundle.yaml | 3 + deployments/porch/3-porch-server.yaml | 21 + deployments/porch/9-controllers.yaml | 25 + func/server/server.go | 4 + go.mod | 6 + go.sum | 4 + internal/telemetry/metrics.go | 203 ++ internal/telemetry/otel.go | 8 + internal/telemetry/pprof.go | 92 + internal/telemetry/pyroscope.go | 103 + pkg/apiserver/config.go | 4 + pkg/externalrepo/git/git.go | 6 + pkg/registry/porch/packagerevision.go | 44 +- .../porch/packagerevision_approval.go | 20 +- .../porch/packagerevisionresources.go | 28 +- scripts/deploy-monitoring.sh | 62 +- test/e2e/performance/README.md | 42 - test/e2e/performance/config.yml | 12 - test/e2e/performance/iterative_test.go | 378 --- test/e2e/performance/iterative_types.go | 61 - test/e2e/performance/performance_suite.go | 229 -- test/e2e/performance/prometheus.go | 84 - test/e2e/performance/prometheus_in_docker.sh | 43 - test/e2e/performance/rules.yml | 47 - test/e2e/performance/utils.go | 43 - test/performance/README.md | 188 +- test/performance/csv_generation.go | 346 +++ test/performance/logger.go | 182 +- test/performance/metrics.go | 217 -- test/performance/performance_suite.go | 995 +++++++ test/performance/porch_metrics_test.go | 904 ++++--- test/performance/prometheus_metrics.go | 67 - test/performance/promql_queries.txt | 125 - test/performance/resources/Kptfile | 13 + test/performance/resources/deployment.yaml | 17 + test/performance/types.go | 34 +- 49 files changed, 8277 insertions(+), 1912 deletions(-) create mode 100644 deployments/metrics-resources/alloy-config.alloy create mode 100644 deployments/metrics-resources/grafana-database-dashboard.json create mode 100644 deployments/metrics-resources/grafana-perf-test-dashboard.json create mode 100644 deployments/metrics-resources/grafana-porch-server-dashboard.json create mode 100644 deployments/metrics-resources/grafana-pyroscope-dashboard.json create mode 100644 deployments/metrics-resources/grafana-resource-usage-dashboard.json create mode 100644 deployments/metrics/postgres-exporter-deployment.yaml create mode 100644 deployments/metrics/pyroscope-deployment.yaml create mode 100644 internal/telemetry/pprof.go create mode 100644 internal/telemetry/pyroscope.go delete mode 100644 test/e2e/performance/README.md delete mode 100644 test/e2e/performance/config.yml delete mode 100644 test/e2e/performance/iterative_test.go delete mode 100644 test/e2e/performance/iterative_types.go delete mode 100644 test/e2e/performance/performance_suite.go delete mode 100644 test/e2e/performance/prometheus.go delete mode 100755 test/e2e/performance/prometheus_in_docker.sh delete mode 100644 test/e2e/performance/rules.yml delete mode 100644 test/e2e/performance/utils.go create mode 100644 test/performance/csv_generation.go delete mode 100644 test/performance/metrics.go create mode 100644 test/performance/performance_suite.go delete mode 100644 test/performance/prometheus_metrics.go delete mode 100644 test/performance/promql_queries.txt create mode 100644 test/performance/resources/Kptfile create mode 100644 test/performance/resources/deployment.yaml diff --git a/.env.template b/.env.template index ef7a87f93..8c1abe5b3 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,4 @@ # PORCH_GHCR_PREFIX_URL=/kptdev/krm-functions-catalog # DOCKERHUB_MIRROR= # DB_MAVEN_MIRROR= +# GRAFANA_ADMIN_PW diff --git a/.gitignore b/.gitignore index 12b27f668..4facf47e5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ vendor/ apiserver.local.config/ /apiserver/porch *.log +*.csv +load_test_results.txt .env # Development artifact path diff --git a/deployments/metrics-resources/alloy-config.alloy b/deployments/metrics-resources/alloy-config.alloy new file mode 100644 index 000000000..e5b59c77e --- /dev/null +++ b/deployments/metrics-resources/alloy-config.alloy @@ -0,0 +1,640 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Scrapes pods that opt in via profiles.grafana.com/* annotations. +// See: https://grafana.com/docs/pyroscope/latest/deploy-kubernetes/helm/#optional-scrape-your-own-workloads-profiles + +discovery.kubernetes "pyroscope_kubernetes" { + role = "pod" +} + +discovery.relabel "kubernetes_pods" { + targets = discovery.kubernetes.pyroscope_kubernetes.targets + + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + } + + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_service_name"] + target_label = "service_name" + } +} + +discovery.relabel "kubernetes_pods_memory_default_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "kubernetes_pods_memory_custom_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_memory" { + targets = concat(discovery.relabel.kubernetes_pods_memory_default_name.output, discovery.relabel.kubernetes_pods_memory_custom_name.output) + forward_to = [pyroscope.write.pyroscope_write.receiver] + + profiling_config { + profile.process_cpu { + enabled = false + } + profile.memory { + enabled = true + } + profile.goroutine { + enabled = false + } + profile.block { + enabled = false + } + profile.mutex { + enabled = false + } + } +} + +discovery.relabel "kubernetes_pods_cpu_default_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "kubernetes_pods_cpu_custom_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_cpu" { + targets = concat(discovery.relabel.kubernetes_pods_cpu_default_name.output, discovery.relabel.kubernetes_pods_cpu_custom_name.output) + forward_to = [pyroscope.write.pyroscope_write.receiver] + + profiling_config { + profile.process_cpu { + enabled = true + } + profile.memory { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.block { + enabled = false + } + profile.mutex { + enabled = false + } + } +} + +discovery.relabel "kubernetes_pods_goroutine_default_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "kubernetes_pods_goroutine_custom_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_goroutine" { + targets = concat(discovery.relabel.kubernetes_pods_goroutine_default_name.output, discovery.relabel.kubernetes_pods_goroutine_custom_name.output) + forward_to = [pyroscope.write.pyroscope_write.receiver] + + profiling_config { + profile.process_cpu { + enabled = false + } + profile.memory { + enabled = false + } + profile.goroutine { + enabled = true + } + profile.block { + enabled = false + } + profile.mutex { + enabled = false + } + } +} + +discovery.relabel "kubernetes_pods_block_default_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "kubernetes_pods_block_custom_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_block" { + targets = concat(discovery.relabel.kubernetes_pods_block_default_name.output, discovery.relabel.kubernetes_pods_block_custom_name.output) + forward_to = [pyroscope.write.pyroscope_write.receiver] + + profiling_config { + profile.process_cpu { + enabled = false + } + profile.memory { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.block { + enabled = true + } + profile.mutex { + enabled = false + } + } +} + +discovery.relabel "kubernetes_pods_mutex_default_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "kubernetes_pods_mutex_custom_name" { + targets = discovery.relabel.kubernetes_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_mutex" { + targets = concat(discovery.relabel.kubernetes_pods_mutex_default_name.output, discovery.relabel.kubernetes_pods_mutex_custom_name.output) + forward_to = [pyroscope.write.pyroscope_write.receiver] + + profiling_config { + profile.process_cpu { + enabled = false + } + profile.memory { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.block { + enabled = false + } + profile.mutex { + enabled = true + } + } +} + +pyroscope.write "pyroscope_write" { + endpoint { + url = "http://pyroscope.porch-monitoring.svc.cluster.local:4040" + } +} diff --git a/deployments/metrics-resources/grafana-database-dashboard.json b/deployments/metrics-resources/grafana-database-dashboard.json new file mode 100644 index 000000000..b07df3621 --- /dev/null +++ b/deployments/metrics-resources/grafana-database-dashboard.json @@ -0,0 +1,407 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "panels": [], + "title": "Query Activity", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Row operations per second: fetched, returned, inserted, updated, and deleted.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 16, "x": 0, "y": 1 }, + "id": 101, + "options": { + "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "fetched", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "returned", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "inserted", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "updated", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "deleted", + "refId": "E" + } + ], + "title": "Rows", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Current queries per second (commits + rollbacks).", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1000 }, + { "color": "red", "value": 10000 } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "refId": "A" + } + ], + "title": "QPS", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "id": 103, + "panels": [], + "title": "Background Writer & Conflicts", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Background writer buffer allocation and checkpoint activity.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "id": 104, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=~\"$instance\",job=~\"$job\"}[5m])", + "legendFormat": "buffers_alloc", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=~\"$instance\",job=~\"$job\"}[5m])", + "legendFormat": "buffers_backend_fsync", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=~\"$instance\",job=~\"$job\"}[5m])", + "legendFormat": "buffers_backend", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=~\"$instance\",job=~\"$job\"}[5m])", + "legendFormat": "buffers_clean", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=~\"$instance\",job=~\"$job\"}[5m])", + "legendFormat": "buffers_checkpoint", + "refId": "E" + } + ], + "title": "Buffers", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Database deadlocks and query conflicts per second.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "id": 105, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "deadlocks", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}[5m]))", + "legendFormat": "conflicts", + "refId": "B" + } + ], + "title": "Conflicts/Deadlocks", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "id": 106, + "panels": [], + "title": "Connections & Cache", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Shared buffer cache hit ratio.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "max": 1, + "min": 0, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "id": 107, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}))", + "legendFormat": "cache hit rate", + "refId": "A" + } + ], + "title": "Cache Hit Ratio", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Number of active backend connections per database.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "id": 108, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\",job=~\"$job\"}", + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "title": "Active Connections", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["porch", "postgresql", "database"], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(up{job=~\"postgres.*\"}, instance)", + "includeAll": true, + "label": "instance", + "name": "instance", + "query": { + "query": "label_values(up{job=~\"postgres.*\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\",job=~\"$job\"}, datname)", + "includeAll": true, + "label": "db", + "name": "db", + "query": { + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\",job=~\"$job\"}, datname)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "type": "query" + }, + { + "current": { "selected": true, "text": "postgres", "value": "postgres" }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(pg_up, job)", + "includeAll": false, + "label": "job", + "name": "job", + "query": { + "query": "label_values(pg_up, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Porch PostgreSQL", + "uid": "porch-postgresql", + "version": 1, + "weekStart": "" +} diff --git a/deployments/metrics-resources/grafana-perf-test-dashboard.json b/deployments/metrics-resources/grafana-perf-test-dashboard.json new file mode 100644 index 000000000..c986ce3ff --- /dev/null +++ b/deployments/metrics-resources/grafana-perf-test-dashboard.json @@ -0,0 +1,2358 @@ +{ + "title": "Porch Performance Test Dashboard", + "tags": [ + "porch", + "performance", + "metrics" + ], + "timezone": "browser", + "schemaVersion": 16, + "version": 0, + "refresh": "5s", + "time": { + "from": "now-5m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "GITEA-REPO-CREATE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GITEA-REPO-CREATE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GITEA-REPO-CREATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GITEA-REPO-CREATE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GITEA-REPO-CREATE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "percentunit", + "label": "Success Rate", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 2, + "title": "GITEA-REPO-CREATE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 3, + "title": "PORCH-REPO-CREATE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 6 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PORCH-REPO-CREATE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PORCH-REPO-CREATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PORCH-REPO-CREATE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PORCH-REPO-CREATE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 4, + "title": "PORCH-REPO-CREATE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 6 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 5, + "title": "REPO-WAIT - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"REPO-WAIT\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"REPO-WAIT\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"REPO-WAIT\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"REPO-WAIT\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 6, + "title": "REPO-WAIT - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"REPO-WAIT\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"REPO-WAIT\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"REPO-WAIT\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 7, + "title": "LIST - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"LIST\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"LIST\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"LIST\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"LIST\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 8, + "title": "LIST - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"LIST\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"LIST\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"LIST\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 9, + "title": "GET - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 24 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 10, + "title": "GET - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 24 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GET\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GET\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GET\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 11, + "title": "GET-PROPOSED - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 30 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-PROPOSED\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET-PROPOSED\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET-PROPOSED\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-PROPOSED\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 12, + "title": "GET-PROPOSED - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 30 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GET-PROPOSED\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GET-PROPOSED\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GET-PROPOSED\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 13, + "title": "GET-RESOURCES - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-RESOURCES\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET-RESOURCES\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET-RESOURCES\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-RESOURCES\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 14, + "title": "GET-RESOURCES - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GET-RESOURCES\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"GET-RESOURCES\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GET-RESOURCES\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 15, + "title": "CREATE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 42 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"CREATE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"CREATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"CREATE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"CREATE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 16, + "title": "CREATE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 42 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"CREATE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"CREATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"CREATE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 19, + "title": "UPDATE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 54 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"UPDATE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"UPDATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"UPDATE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"UPDATE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 20, + "title": "UPDATE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 54 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"UPDATE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"UPDATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"UPDATE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 23, + "title": "PROPOSE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 66 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PROPOSE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PROPOSE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 24, + "title": "PROPOSE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 66 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"PROPOSE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 25, + "title": "APPROVE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 72 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"APPROVE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"APPROVE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"APPROVE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"APPROVE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 26, + "title": "APPROVE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 72 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"APPROVE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"APPROVE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"APPROVE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 27, + "title": "PROPOSE-DELETION - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 78 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE-DELETION\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PROPOSE-DELETION\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PROPOSE-DELETION\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE-DELETION\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 28, + "title": "PROPOSE-DELETION - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 78 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 29, + "title": "DELETE - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 84 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"DELETE\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"DELETE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"DELETE\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"DELETE\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 30, + "title": "DELETE - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 84 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"DELETE\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"DELETE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"DELETE\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 31, + "title": "Overall Success Rate by Operation", + "type": "graph", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 90 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{status=\"success\"}) by (operation) / sum(porch_perf_operations_total) by (operation)", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "yaxes": [ + { + "format": "percentunit", + "label": "Success Rate", + "min": 0, + "max": 1, + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 32, + "title": "All Operations - Avg Duration Comparison", + "type": "graph", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 98 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (operation) (rate(porch_perf_operation_duration_seconds_sum[5m])) / sum by (operation) (rate(porch_perf_operation_duration_seconds_count[5m]))", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Avg Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 33, + "title": "All Operations - Total Count", + "type": "graph", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 98 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total) by (operation)", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "yaxes": [ + { + "format": "short", + "label": "Total Operations", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 34, + "title": "Lifecycle Transition Duration", + "type": "graph", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 106 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Draft\", to_state=\"Proposed\"}[5m])) by (le))", + "legendFormat": "Draft -> Proposed (p99)", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Draft\", to_state=\"Proposed\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Draft\", to_state=\"Proposed\"}[5m]))", + "legendFormat": "Draft -> Proposed (avg)", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Proposed\", to_state=\"Published\"}[5m])) by (le))", + "legendFormat": "Proposed -> Published (p99)", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Proposed\", to_state=\"Published\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Proposed\", to_state=\"Published\"}[5m]))", + "legendFormat": "Proposed -> Published (avg)", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Published\", to_state=\"DeletionProposed\"}[5m])) by (le))", + "legendFormat": "Published -> DeletionProposed (p99)", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Published\", to_state=\"DeletionProposed\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Published\", to_state=\"DeletionProposed\"}[5m]))", + "legendFormat": "Published -> DeletionProposed (avg)", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Proposed\", to_state=\"DeletionProposed\"}[5m])) by (le))", + "legendFormat": "Proposed -> DeletionProposed (p99)", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Proposed\", to_state=\"DeletionProposed\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Proposed\", to_state=\"DeletionProposed\"}[5m]))", + "legendFormat": "Proposed -> DeletionProposed (avg)", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"DeletionProposed\", to_state=\"deleted\"}[5m])) by (le))", + "legendFormat": "DeletionProposed -> deleted (p99)", + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"DeletionProposed\", to_state=\"deleted\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"DeletionProposed\", to_state=\"deleted\"}[5m]))", + "legendFormat": "DeletionProposed -> deleted (avg)", + "refId": "K" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 35, + "title": "Active Operations", + "type": "graph", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 106 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_active_operations) by (operation)", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "yaxes": [ + { + "format": "short", + "label": "Active Count", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 36, + "title": "Total Repositories Created", + "type": "stat", + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 114 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "porch_perf_repositories_created_total", + "refId": "A" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "displayName": "Repositories" + } + } + }, + { + "id": 37, + "title": "Total Packages Created", + "type": "stat", + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 114 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "porch_perf_packages_created_total", + "refId": "A" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "displayName": "Packages" + } + } + }, + { + "id": 38, + "title": "Package Revisions by Status", + "type": "stat", + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 114 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_package_revisions_total) by (operation, status)", + "legendFormat": "{{operation}} - {{status}}", + "refId": "A" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + } + } + ], + "templating": { + "list": [ + { + "name": "namespace", + "type": "query", + "query": "label_values(porch_perf_test_run_info, namespace)", + "current": { + "text": "porch-metrics", + "value": "porch-metrics" + }, + "refresh": 1 + } + ] + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + } +} \ No newline at end of file diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json new file mode 100644 index 000000000..e1f2db3cd --- /dev/null +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -0,0 +1,500 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "panels": [], + "title": "PackageRevisions API Calls", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions GET latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "id": 101, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"GET\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"GET\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"GET\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"GET\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PR GET", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions LIST latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "id": 102, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"LIST\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"LIST\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"LIST\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"LIST\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PR LIST", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions CREATE latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "id": 103, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"CREATE\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"CREATE\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"CREATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"CREATE\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PR CREATE", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions UPDATE latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, + "id": 104, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PR UPDATE", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions DELETE latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, + "id": 105, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"DELETE\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"DELETE\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"DELETE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"DELETE\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PR DELETE", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "id": 200, + "panels": [], + "title": "PackageRevisionResources API Calls", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisionResources GET latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, + "id": 201, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PRR GET", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisionResources UPDATE latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, + "id": 202, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PRR UPDATE", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisionResources LIST latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, + "id": 203, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "PRR LIST", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, + "id": 300, + "panels": [], + "title": "Approval API Calls", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions Approval GET latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, + "id": 301, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "Approval GET", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "PackageRevisions Approval UPDATE latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, + "id": 302, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "Approval UPDATE", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, + "id": 500, + "panels": [], + "title": "Request Counts by User", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of PackageRevision requests per operation and user.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, + "id": 501, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_requests_by_user_total{resource=\"PackageRevision\"}[5m])) by (op, user)", + "legendFormat": "{{op}} / {{user}}", + "refId": "A" + } + ], + "title": "PackageRevision Requests/s by Operation & User", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of ExternalRepo requests per operation and user.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, + "id": 502, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(porch_api_requests_by_user_total{resource=\"ExternalRepo\"}[5m])) by (op, user)", + "legendFormat": "{{op}} / {{user}}", + "refId": "A" + } + ], + "title": "ExternalRepo Requests/s by Operation & User", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Total cumulative request counts broken down by resource, operation, and user.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }, + "id": 503, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(porch_api_requests_by_user_total) by (resource, op, user)", + "legendFormat": "{{resource}} / {{op}} / {{user}}", + "refId": "A" + } + ], + "title": "All Resources — Total Request Count by Resource, Operation & User", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["porch", "api"], + "templating": { "list": [] }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Porch API", + "uid": "porch-api-queues", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/deployments/metrics-resources/grafana-pyroscope-dashboard.json b/deployments/metrics-resources/grafana-pyroscope-dashboard.json new file mode 100644 index 000000000..8d369ab5b --- /dev/null +++ b/deployments/metrics-resources/grafana-pyroscope-dashboard.json @@ -0,0 +1,114 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": {}, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "groupBy": [], + "labelSelector": "{service_name=\"$service\"}", + "profileTypeId": "$profiletype", + "queryType": "profile", + "refId": "A", + "spanSelector": [] + } + ], + "title": "Porch – $service · $profiletype", + "type": "flamegraph" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 42, + "tags": ["porch", "pyroscope", "profiling"], + "templating": { + "list": [ + { + "allowCustomValue": false, + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "definition": "", + "name": "profiletype", + "options": [], + "query": { + "type": "profileType" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "definition": "", + "name": "service", + "options": [], + "query": { + "labelName": "service_name", + "profileTypeId": "$profiletype", + "type": "labelValue" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Pyroscope – Porch profiling", + "uid": "pyroscope-porch", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/deployments/metrics-resources/grafana-resource-usage-dashboard.json b/deployments/metrics-resources/grafana-resource-usage-dashboard.json new file mode 100644 index 000000000..dc63a0d23 --- /dev/null +++ b/deployments/metrics-resources/grafana-resource-usage-dashboard.json @@ -0,0 +1,1196 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "panels": [], + "title": "CPU Usage", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "CPU usage rate for porch-server", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "CPU cores", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, + "id": 2, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-server\"}[5m])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Porch Server - CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "CPU usage rate for function-runner", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "CPU cores", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, + "id": 3, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"function-runner\"}[5m])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Function Runner - CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "CPU usage for porch-controllers (shared process for repositories, packagerevisions, packagevariants, packagevariantsets, and functionconfigs reconcilers)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "CPU cores", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, + "id": 16, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Porch Controllers - CPU Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "id": 4, + "panels": [], + "title": "Memory Usage", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Memory usage for porch-server", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Memory", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 }, + "id": 5, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_resident_memory_bytes{job=\"porch-server\"}", + "legendFormat": "resident - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_virtual_memory_bytes{job=\"porch-server\"}", + "legendFormat": "virtual - {{instance}}", + "refId": "B" + } + ], + "title": "Porch Server - Memory Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Memory usage for function-runner", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Memory", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 }, + "id": 6, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_resident_memory_bytes{job=\"function-runner\"}", + "legendFormat": "resident - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_virtual_memory_bytes{job=\"function-runner\"}", + "legendFormat": "virtual - {{instance}}", + "refId": "B" + } + ], + "title": "Function Runner - Memory Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Memory usage for porch-controllers (shared process for all reconcilers)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Memory", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 }, + "id": 17, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", + "legendFormat": "resident - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_virtual_memory_bytes{job=\"porch-controllers\"}", + "legendFormat": "virtual - {{instance}}", + "refId": "B" + } + ], + "title": "Porch Controllers - Memory Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "id": 7, + "panels": [], + "title": "Go Runtime Metrics", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Number of active goroutines", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Goroutines", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 19 }, + "id": 8, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_goroutines{job=\"porch-server\"}", + "legendFormat": "porch-server ({{instance}})", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_goroutines{job=\"function-runner\"}", + "legendFormat": "function-runner ({{instance}})", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_goroutines{job=\"porch-controllers\"}", + "legendFormat": "porch-controllers ({{instance}})", + "refId": "C" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Go heap memory allocation", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Memory", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 19 }, + "id": 9, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_memstats_heap_alloc_bytes{job=\"porch-server\"}", + "legendFormat": "porch-server heap alloc ({{instance}})", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_memstats_heap_inuse_bytes{job=\"porch-server\"}", + "legendFormat": "porch-server heap inuse ({{instance}})", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_memstats_heap_alloc_bytes{job=\"function-runner\"}", + "legendFormat": "function-runner heap alloc ({{instance}})", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_memstats_heap_inuse_bytes{job=\"function-runner\"}", + "legendFormat": "function-runner heap inuse ({{instance}})", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_memstats_heap_alloc_bytes{job=\"porch-controllers\"}", + "legendFormat": "porch-controllers heap alloc ({{instance}})", + "refId": "E" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_memstats_heap_inuse_bytes{job=\"porch-controllers\"}", + "legendFormat": "porch-controllers heap inuse ({{instance}})", + "refId": "F" + } + ], + "title": "Go Heap Memory", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Garbage collection duration", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Duration", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 19 }, + "id": 10, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(go_gc_duration_seconds_sum{job=\"porch-server\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"porch-server\"}[5m])", + "legendFormat": "porch-server avg GC ({{instance}})", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(go_gc_duration_seconds_sum{job=\"function-runner\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"function-runner\"}[5m])", + "legendFormat": "function-runner avg GC ({{instance}})", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(go_gc_duration_seconds_sum{job=\"porch-controllers\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"porch-controllers\"}[5m])", + "legendFormat": "porch-controllers avg GC ({{instance}})", + "refId": "C" + } + ], + "title": "GC Duration (avg)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "id": 11, + "panels": [], + "title": "Resource Stats Summary", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 28 }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-server\"}[5m])", + "refId": "A" + } + ], + "title": "Porch Server CPU (cores)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 536870912 }, + { "color": "red", "value": 1073741824 } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 28 }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_resident_memory_bytes{job=\"porch-server\"}", + "refId": "A" + } + ], + "title": "Porch Server Memory", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 28 }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"function-runner\"}[5m])", + "refId": "A" + } + ], + "title": "Function Runner CPU (cores)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 536870912 }, + { "color": "red", "value": 1073741824 } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 28 }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_resident_memory_bytes{job=\"function-runner\"}", + "refId": "A" + } + ], + "title": "Function Runner Memory", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 28 }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", + "refId": "A" + } + ], + "title": "Porch Controllers CPU (cores)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 536870912 }, + { "color": "red", "value": 1073741824 } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 28 }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", + "refId": "A" + } + ], + "title": "Porch Controllers Memory", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "id": 20, + "panels": [], + "title": "Porch Controllers - Per-Controller Workload", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Reconciliation backlog per controller (repositories=repository, packagerevisions=packagerevision, packagevariants, packagevariantsets, functionconfigs=functionconfig)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Items", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 33 }, + "id": 21, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "workqueue_depth{job=\"porch-controllers\", name=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Workqueue Depth (all controllers)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Reconciliations per second per controller", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Reconciles/s", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 33 }, + "id": 22, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"porch-controllers\", controller=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}[5m])) by (controller)", + "legendFormat": "{{controller}}", + "refId": "A" + } + ], + "title": "Reconcile Rate (all controllers)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "p99 reconciliation duration per controller", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Duration", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 33 }, + "id": 23, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=\"porch-controllers\", controller=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}[5m])) by (le, controller))", + "legendFormat": "{{controller}} p99", + "refId": "A" + } + ], + "title": "Reconcile Duration p99 (all controllers)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Filtered view for the selected controller ($controller)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Items", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, + "id": 24, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "workqueue_depth{job=\"porch-controllers\", name=\"$controller\"}", + "legendFormat": "depth", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"porch-controllers\", controller=\"$controller\"}[5m])) by (result)", + "legendFormat": "reconcile {{result}}", + "refId": "B" + } + ], + "title": "$controller - Workload", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Shared porch-controllers process resource usage (same for all reconcilers)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, + "id": 25, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", + "legendFormat": "CPU cores", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "go_goroutines{job=\"porch-controllers\"}", + "legendFormat": "goroutines", + "refId": "B" + } + ], + "title": "Porch Controllers - Shared Process Resources", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 }, + "id": 30, + "panels": [], + "title": "PostgreSQL Resource Usage", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "CPU usage for the porch-postgresql container (from cAdvisor via kubelet). postgres-exporter provides database query metrics only, not container CPU.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "CPU cores", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 }, + "id": 31, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[5m])) by (pod)", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "PostgreSQL - CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Memory working set for the porch-postgresql container (from cAdvisor via kubelet).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Memory", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, + "id": 32, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "container_memory_working_set_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}", + "legendFormat": "working set - {{pod}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "container_memory_usage_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}", + "legendFormat": "usage - {{pod}}", + "refId": "B" + } + ], + "title": "PostgreSQL - Memory Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Current CPU usage for porch-postgresql (limit: 500m per deployment)", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.25 }, + { "color": "red", "value": 0.5 } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 58 }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "PostgreSQL CPU (cores)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Current memory working set for porch-postgresql (limit: 512Mi per deployment)", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 268435456 }, + { "color": "red", "value": 536870912 } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 58 }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"})", + "refId": "A" + } + ], + "title": "PostgreSQL Memory", + "type": "stat" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": ["porch", "resources", "kubernetes"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "repository", "value": "repository" }, + "hide": 0, + "includeAll": false, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [ + { "selected": true, "text": "repository", "value": "repository" }, + { "selected": false, "text": "packagerevision", "value": "packagerevision" }, + { "selected": false, "text": "packagevariant", "value": "packagevariant" }, + { "selected": false, "text": "packagevariantset", "value": "packagevariantset" }, + { "selected": false, "text": "functionconfig", "value": "functionconfig" } + ], + "query": "repository,packagerevision,packagevariant,packagevariantset,functionconfig", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Porch Resource Usage", + "uid": "porch-resource-usage", + "version": 4, + "weekStart": "" +} diff --git a/deployments/metrics-resources/prometheus-config.yaml b/deployments/metrics-resources/prometheus-config.yaml index 0ef2d3f9e..5004e5283 100644 --- a/deployments/metrics-resources/prometheus-config.yaml +++ b/deployments/metrics-resources/prometheus-config.yaml @@ -45,4 +45,45 @@ scrape_configs: - targets: ['function-runner.porch-system.svc.cluster.local:9464'] labels: service: 'function-runner' - component: 'grpc-server' \ No newline at end of file + component: 'grpc-server' + + - job_name: 'postgres' + scrape_interval: 10s + scrape_timeout: 10s + static_configs: + - targets: ['postgres-exporter.porch-monitoring.svc.cluster.local:9187'] + labels: + service: 'porch' + component: 'postgresql' + + # Container CPU/memory for porch-postgresql (and other porch-system pods). + - job_name: 'kubernetes-cadvisor' + honor_labels: true + scrape_interval: 10s + scrape_timeout: 10s + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # Perf tests run on the host (go test) and expose OTel metrics on port 9095. + # On kind/Linux, Prometheus inside the cluster reaches the host via the Docker bridge gateway. + - job_name: 'porch-performance-tests' + scrape_interval: 5s + scrape_timeout: 5s + static_configs: + - targets: ['172.17.0.1:9095'] + labels: + service: 'porch-perf-test' \ No newline at end of file diff --git a/deployments/metrics/grafana-deployment.yaml b/deployments/metrics/grafana-deployment.yaml index 86e0a85a8..531e33925 100644 --- a/deployments/metrics/grafana-deployment.yaml +++ b/deployments/metrics/grafana-deployment.yaml @@ -135,6 +135,12 @@ data: url: http://prometheus:9090 isDefault: true editable: true + - name: Pyroscope + type: grafana-pyroscope-datasource + access: proxy + uid: pyroscope + url: http://pyroscope:4040 + editable: true --- apiVersion: v1 data: diff --git a/deployments/metrics/postgres-exporter-deployment.yaml b/deployments/metrics/postgres-exporter-deployment.yaml new file mode 100644 index 000000000..8648074e4 --- /dev/null +++ b/deployments/metrics/postgres-exporter-deployment.yaml @@ -0,0 +1,98 @@ +# Copyright 2026 The kpt Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: postgres-exporter +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter + template: + metadata: + labels: + app: postgres-exporter + spec: + automountServiceAccountToken: false + containers: + - name: postgres-exporter + image: docker.io/prometheuscommunity/postgres-exporter:v0.16.0 # kpt-set: ${postgres-exporter-image} + imagePullPolicy: IfNotPresent + env: + - name: DATA_SOURCE_USER + valueFrom: + secretKeyRef: + name: postgres-exporter-db-creds + key: DB_USER + - name: DATA_SOURCE_PASS + valueFrom: + secretKeyRef: + name: postgres-exporter-db-creds + key: DB_PASSWORD + - name: DATA_SOURCE_URI + value: "porch-postgresql.porch-system.svc.cluster.local:5432/porch?sslmode=disable" # kpt-set: ${postgres-exporter-data-source-uri} + ports: + - name: metrics + containerPort: 9187 # kpt-set: ${postgres-exporter-container-port} + livenessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: postgres-exporter +spec: + type: ClusterIP + ports: + - name: metrics + port: 9187 # kpt-set: ${postgres-exporter-container-port} + targetPort: metrics + selector: + app: postgres-exporter +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-exporter-db-creds + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: postgres-exporter +type: Opaque +data: + DB_USER: cG9yY2g= # kpt-set: ${postgres-exporter-db-user} + DB_PASSWORD: cG9yY2g= # kpt-set: ${postgres-exporter-db-password} diff --git a/deployments/metrics/pyroscope-deployment.yaml b/deployments/metrics/pyroscope-deployment.yaml new file mode 100644 index 000000000..f1f72ea6f --- /dev/null +++ b/deployments/metrics/pyroscope-deployment.yaml @@ -0,0 +1,144 @@ +# Copyright 2026 The Nephio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: pyroscope + namespace: porch-monitoring # kpt-set: ${namespace} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pyroscope + namespace: porch-monitoring # kpt-set: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app: pyroscope + template: + metadata: + labels: + app: pyroscope + spec: + serviceAccountName: pyroscope + containers: + - name: pyroscope + image: docker.io/grafana/pyroscope:latest # kpt-set: ${pyroscope-image} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 4040 + name: ingest + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + +--- +apiVersion: v1 +kind: Service +metadata: + name: pyroscope + namespace: porch-monitoring # kpt-set: ${namespace} +spec: + ports: + - port: 4040 + protocol: TCP + targetPort: 4040 + name: ingest + selector: + app: pyroscope + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alloy + namespace: porch-monitoring # kpt-set: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app: alloy + template: + metadata: + labels: + app: alloy + spec: + serviceAccountName: alloy + containers: + - name: alloy + image: docker.io/grafana/alloy:latest # kpt-set: ${alloy-image} + imagePullPolicy: IfNotPresent + args: + - run + - /etc/alloy/config.alloy + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + ports: + - containerPort: 12345 + name: http + volumeMounts: + - name: alloy-config + mountPath: /etc/alloy + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + volumes: + - name: alloy-config + configMap: + name: alloy-config + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alloy + namespace: porch-monitoring # kpt-set: ${namespace} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: alloy +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: alloy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: alloy +subjects: + - kind: ServiceAccount + name: alloy + namespace: porch-monitoring # kpt-set: ${namespace} diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index 8a0210766..874630dc0 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -31,6 +31,18 @@ spec: metadata: labels: app: function-runner + annotations: + profiles.grafana.com/service_name: porch-function-runner + profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/cpu.port_name: "pprof" + profiles.grafana.com/memory.scrape: "true" + profiles.grafana.com/memory.port_name: "pprof" + profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/goroutine.port_name: "pprof" + profiles.grafana.com/block.scrape: "true" + profiles.grafana.com/block.port_name: "pprof" + profiles.grafana.com/mutex.scrape: "true" + profiles.grafana.com/mutex.port_name: "pprof" spec: serviceAccountName: porch-fn-runner containers: @@ -62,10 +74,14 @@ spec: value: none - name: OTEL_SERVICE_NAME value: porch-function-runner + - name: PORCH_PPROF_PORT + value: "8080" ports: - containerPort: 9445 - containerPort: 9464 name: metrics + - containerPort: 8080 + name: pprof # Add grpc readiness probe to ensure the cache is ready startupProbe: exec: @@ -118,3 +134,7 @@ spec: protocol: TCP targetPort: 9464 name: metrics + - port: 8080 + protocol: TCP + targetPort: 8080 + name: pprof diff --git a/deployments/porch/3-porch-postgres-bundle.yaml b/deployments/porch/3-porch-postgres-bundle.yaml index a579df45d..2306da015 100644 --- a/deployments/porch/3-porch-postgres-bundle.yaml +++ b/deployments/porch/3-porch-postgres-bundle.yaml @@ -499,3 +499,6 @@ data: REFERENCES package_revisions (k8s_name_space, k8s_name) ON DELETE CASCADE ); + + -- Required for prometheus-community/postgres-exporter to collect metrics. + GRANT pg_monitor TO porch; diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index aa9b2f797..4108579e8 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -31,6 +31,18 @@ spec: metadata: labels: app: porch-server + annotations: + profiles.grafana.com/service_name: porch-server + profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/cpu.port_name: "pprof" + profiles.grafana.com/memory.scrape: "true" + profiles.grafana.com/memory.port_name: "pprof" + profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/goroutine.port_name: "pprof" + profiles.grafana.com/block.scrape: "true" + profiles.grafana.com/block.port_name: "pprof" + profiles.grafana.com/mutex.scrape: "true" + profiles.grafana.com/mutex.port_name: "pprof" spec: serviceAccountName: porch-server volumes: @@ -82,6 +94,8 @@ spec: value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER value: none + - name: PORCH_PPROF_PORT + value: "8080" args: - --function-runner=function-runner:9445 - --cache-directory=/cache @@ -96,6 +110,9 @@ spec: - containerPort: 9464 name: metrics protocol: TCP + - containerPort: 8080 + name: pprof + protocol: TCP startupProbe: httpGet: path: /healthz @@ -143,5 +160,9 @@ spec: protocol: TCP targetPort: 9464 name: metrics + - port: 8080 + protocol: TCP + targetPort: 8080 + name: pprof selector: app: porch-server diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index f0fad9b9f..573de63de 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -33,6 +33,18 @@ spec: metadata: labels: k8s-app: porch-controllers + annotations: + profiles.grafana.com/service_name: porch-controllers + profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/cpu.port_name: "pprof" + profiles.grafana.com/memory.scrape: "true" + profiles.grafana.com/memory.port_name: "pprof" + profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/goroutine.port_name: "pprof" + profiles.grafana.com/block.scrape: "true" + profiles.grafana.com/block.port_name: "pprof" + profiles.grafana.com/mutex.scrape: "true" + profiles.grafana.com/mutex.port_name: "pprof" spec: serviceAccountName: porch-controllers securityContext: @@ -80,6 +92,8 @@ spec: value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER value: none + - name: PORCH_PPROF_PORT + value: "8080" envFrom: - configMapRef: name: porch-db-config @@ -121,6 +135,13 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + ports: + - containerPort: 9464 + name: metrics + protocol: TCP + - containerPort: 8080 + name: pprof + protocol: TCP --- apiVersion: v1 kind: Service @@ -137,5 +158,9 @@ spec: protocol: TCP targetPort: 9443 name: webhooks + - port: 8080 + protocol: TCP + targetPort: 8080 + name: pprof selector: k8s-app: porch-controllers diff --git a/func/server/server.go b/func/server/server.go index df08afb62..e9e831e06 100644 --- a/func/server/server.go +++ b/func/server/server.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client/config" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -244,6 +245,9 @@ func buildFnConfigReconciler(o *options, scheme *runtime.Scheme) (*functionconfi mgr, err := ctrl.NewManager(restCfg, ctrl.Options{ Scheme: scheme, Cache: cacheOpts, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, }) if err != nil { return nil, err diff --git a/go.mod b/go.mod index 8af4247ef..0828f111f 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.21.6 github.com/google/uuid v1.6.0 + github.com/grafana/pyroscope-go v1.4.1 github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 github.com/kptdev/kpt v1.0.0-beta.67 @@ -72,6 +73,11 @@ require ( require sigs.k8s.io/cli-utils v0.37.2 // indirect +require ( + github.com/grafana/pyroscope-go/godeltaprof v0.1.11 // indirect + sigs.k8s.io/cli-utils v0.37.2 // indirect +) + require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go/auth v0.20.0 // indirect diff --git a/go.sum b/go.sum index d924893e3..2d81cd6b3 100644 --- a/go.sum +++ b/go.sum @@ -229,6 +229,10 @@ github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/grafana/pyroscope-go v1.4.1 h1:SKvuZz1qTFpNXjHY3XGlm92mS9wJeCI/TZR42XcDs9w= +github.com/grafana/pyroscope-go v1.4.1/go.mod h1:WQY21vHNiD2o/icxqVPM0AmofabycbhnPPHoV1/4X6s= +github.com/grafana/pyroscope-go/godeltaprof v0.1.11 h1:el5LYpXissAiCKZ5/6yjlr6mhYVV6Cp5lahTocxraXM= +github.com/grafana/pyroscope-go/godeltaprof v0.1.11/go.mod h1:jl1V8M4cWsXciROCPIDDG7CtjSjT/ECbp6eLVuMxYRI= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index 91ac2c2d3..ed181c38f 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -16,24 +16,60 @@ package telemetry import ( "context" + "fmt" + "time" "github.com/kptdev/porch/pkg/repository" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" ) const meterName = "github.com/kptdev/porch" var ( + apiCallDurationSeconds metric.Float64Histogram + RequestsTotal metric.Float64Counter prResourceSizeHistogram metric.Int64Histogram prResourceSizeGauge metric.Int64Gauge + + // Performance tests related vars + perfOperationDuration metric.Float64Histogram + perfOperationCounter metric.Float64Counter + perfRepositoryCounter metric.Float64Counter + perfPackageCounter metric.Float64Counter + perfPackageRevisionCounter metric.Float64Counter + perfLifecycleTransitionDuration metric.Float64Histogram + perfTestRunInfoGauge metric.Float64Gauge + perfActiveOperations metric.Float64UpDownCounter ) func InitMetrics() (err error) { m := otel.Meter(meterName) + apiCallDurationSeconds, err = m.Float64Histogram( + "porch_api_call_duration_seconds", + metric.WithDescription("Duration of porch API calls in seconds."), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries( + 0.001, 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, + 0.256, 0.512, 1.024, 2.048, 4.096, 8.192, 16.384, 32.768, + ), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_api_call_duration_seconds: %v", err)) + } + + RequestsTotal, err = m.Float64Counter( + "porch_api_requests_by_user", + metric.WithDescription("Total number of requests tracked by BurstCounter, broken down by resource, operation, and user."), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_api_requests_by_user: %v", err)) + } + prResourceSizeHistogram, err = m.Int64Histogram( "porch_package_size_bytes", metric.WithUnit("By"), @@ -55,10 +91,105 @@ func InitMetrics() (err error) { return } + // Performance test related metrics + perfOperationDuration, err = m.Float64Histogram( + "porch_perf_operation_duration_seconds", + metric.WithDescription("Duration of Porch performance test operations in seconds"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30, 60, 120), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_operation_duration_seconds: %v", err)) + } + + perfOperationCounter, err = m.Float64Counter( + "porch_perf_operations_total", + metric.WithDescription("Total number of Porch performance test operations"), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_operations_total: %v", err)) + } + + perfRepositoryCounter, err = m.Float64Counter( + "porch_perf_repositories_created_total", + metric.WithDescription("Total number of repositories created in performance tests"), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_repositories_created_total: %v", err)) + } + + perfPackageCounter, err = m.Float64Counter( + "porch_perf_packages_created_total", + metric.WithDescription("Total number of packages created in performance tests"), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_packages_created_total: %v", err)) + } + + perfPackageRevisionCounter, err = m.Float64Counter( + "porch_perf_package_revisions_total", + metric.WithDescription("Total number of package revisions created in performance tests"), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_package_revisions_total: %v", err)) + } + + perfLifecycleTransitionDuration, err = m.Float64Histogram( + "porch_perf_lifecycle_transition_duration_seconds", + metric.WithDescription("Duration of package lifecycle transitions in seconds"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.1, 0.5, 1, 2, 5, 10, 30, 60), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_lifecycle_transition_duration_seconds: %v", err)) + } + + perfTestRunInfoGauge, err = m.Float64Gauge( + "porch_perf_test_run_info", + metric.WithDescription("Information about the current performance test run"), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_test_run_info: %v", err)) + } + + perfActiveOperations, err = m.Float64UpDownCounter( + "porch_perf_active_operations", + metric.WithDescription("Number of currently active operations"), + ) + if err != nil { + panic(fmt.Sprintf("failed to create porch_perf_active_operations: %v", err)) + } + return nil } // Porch server and function runner metric recording functions +func RecordAPICallDuration(resource, verb string, durationSeconds float64) { + if apiCallDurationSeconds == nil { + return + } + apiCallDurationSeconds.Record(context.Background(), durationSeconds, + metric.WithAttributes( + attribute.String("resource", resource), + attribute.String("verb", verb), + ), + ) +} + +func RecordRequestCount(ctx context.Context, resource, op string) { + if RequestsTotal == nil { + return + } + user := getK8sUserName(ctx) + RequestsTotal.Add(context.Background(), 1, + metric.WithAttributes( + attribute.String("resource", resource), + attribute.String("op", op), + attribute.String("user", user), + ), + ) +} + func RecordPackageRevisionResourcesSize(ctx context.Context, prKey repository.PackageRevisionKey, resourcesSize int64) { prPath := func() string { if prKey.PKey().Path != "" { @@ -92,3 +223,75 @@ func RecordPackageRevisionResourcesSize(ctx context.Context, prKey repository.Pa } prResourceSizeGauge.Record(ctx, resourcesSize, metric.WithAttributeSet(attributes)) } + +// Performance test metric recording functions +func PerfTestRecordMetric(operation, repoName, pkgName string, duration time.Duration, err error) { + attrs := metric.WithAttributes( + attribute.String("operation", operation), + attribute.String("repository", repoName), + attribute.String("package", pkgName), + attribute.String("status", statusLabel(err)), + ) + ctx := context.Background() + perfOperationDuration.Record(ctx, duration.Seconds(), attrs) + perfOperationCounter.Add(ctx, 1, attrs) +} + +func PerfTestRecordLifecycleTransition(fromState, toState, repoName, pkgName string, duration time.Duration, err error) { + perfLifecycleTransitionDuration.Record(context.Background(), duration.Seconds(), + metric.WithAttributes( + attribute.String("from_state", fromState), + attribute.String("to_state", toState), + attribute.String("repository", repoName), + attribute.String("package", pkgName), + attribute.String("status", statusLabel(err)), + ), + ) +} + +func PerfTestRecordPackageRevision(operation string, err error) { + perfPackageRevisionCounter.Add(context.Background(), 1, + metric.WithAttributes( + attribute.String("operation", operation), + attribute.String("status", statusLabel(err)), + ), + ) +} + +func PerfTestSetTestRunInfo(testName, namespace string, startTime time.Time) { + perfTestRunInfoGauge.Record(context.Background(), 1, + metric.WithAttributes( + attribute.String("test_name", testName), + attribute.String("namespace", namespace), + attribute.String("start_time", startTime.Format(time.RFC3339)), + ), + ) +} + +func PerfTestRecordActiveOperation(operation string, delta float64) { + perfActiveOperations.Add(context.Background(), delta, + metric.WithAttributes(attribute.String("operation", operation)), + ) +} + +func PerfTestIncrementRepositoryCounter() { + perfRepositoryCounter.Add(context.Background(), 1) +} + +func PerfTestIncrementPackageCounter() { + perfPackageCounter.Add(context.Background(), 1) +} + +func statusLabel(err error) string { + if err != nil { + return "error" + } + return "success" +} + +func getK8sUserName(ctx context.Context) string { + if user, ok := request.UserFrom(ctx); ok { + return user.GetName() + } + return "" +} diff --git a/internal/telemetry/otel.go b/internal/telemetry/otel.go index 0b28c99fd..8d5203ea0 100644 --- a/internal/telemetry/otel.go +++ b/internal/telemetry/otel.go @@ -52,6 +52,7 @@ type OTelResources struct { metricsPort int meterProvider *sdkmetric.MeterProvider tracerProvider *trace.TracerProvider + profiling *Profiling } // Shutdown gracefully shuts down all OpenTelemetry resources. @@ -73,6 +74,9 @@ func (r *OTelResources) Shutdown(ctx context.Context) error { errs = append(errs, fmt.Errorf("tracer provider shutdown: %w", err)) } } + if r.profiling != nil { + r.profiling.Stop() + } if len(errs) > 0 { return fmt.Errorf("otel shutdown errors: %v", errs) } @@ -116,6 +120,10 @@ func SetupOpenTelemetry(ctx context.Context) (*OTelResources, error) { return nil, err } + prof := &Profiling{} + prof.Start() + res.profiling = prof + http.DefaultTransport = otelhttp.NewTransport(http.DefaultTransport) http.DefaultClient.Transport = http.DefaultTransport klog.Infof("OpenTelemetry initialized in %s", time.Since(setupTiming)) diff --git a/internal/telemetry/pprof.go b/internal/telemetry/pprof.go new file mode 100644 index 000000000..63331d7a3 --- /dev/null +++ b/internal/telemetry/pprof.go @@ -0,0 +1,92 @@ +// Copyright 2025 The Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + "fmt" + "net/http" + httppprof "net/http/pprof" + "os" + "runtime" + "strconv" + "time" + + "k8s.io/klog/v2" +) + +const PProfPortEnvVar = "PORCH_PPROF_PORT" + +type Profiling struct { + port int + server *http.Server +} + +func (p *Profiling) Start() { + if envport := os.Getenv(PProfPortEnvVar); envport != "" { + parsed, err := strconv.Atoi(envport) + if err != nil { + klog.Warningf("Failed to parse %s environment variable (%s) as int: %v", PProfPortEnvVar, envport, err) + return + } + p.port = parsed + } else { + return + } + + // Enable profiling for mutex and block profiles + runtime.SetMutexProfileFraction(1) + runtime.SetBlockProfileRate(1) + + mux := http.NewServeMux() + + mux.HandleFunc("/debug/pprof/", httppprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", httppprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", httppprof.Trace) + mux.Handle("/debug/pprof/heap", httppprof.Handler("heap")) + mux.Handle("/debug/pprof/goroutine", httppprof.Handler("goroutine")) + mux.Handle("/debug/pprof/threadcreate", httppprof.Handler("threadcreate")) + mux.Handle("/debug/pprof/block", httppprof.Handler("block")) + mux.Handle("/debug/pprof/mutex", httppprof.Handler("mutex")) + mux.Handle("/debug/pprof/allocs", httppprof.Handler("allocs")) + p.server = &http.Server{ + Addr: fmt.Sprintf(":%d", p.port), + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } + go p.serve() +} + +func (p *Profiling) Stop() { + if p.server != nil { + klog.Infof("Shutting down profiling server") + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(3*time.Second)) + defer cancel() + err := p.server.Shutdown(ctx) + if err != nil { + klog.Errorf("Error shutting down profiling server: %v", err) + } + } +} + +func (p *Profiling) serve() { + klog.Infof("Starting profiling server on port :%d", p.port) + if err := p.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + klog.Errorf("Error starting metrics server: %v", err) + } + klog.Info("Profiling server stopped") +} diff --git a/internal/telemetry/pyroscope.go b/internal/telemetry/pyroscope.go new file mode 100644 index 000000000..afb9bebc3 --- /dev/null +++ b/internal/telemetry/pyroscope.go @@ -0,0 +1,103 @@ +// Copyright 2026 The Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "os" + "runtime" + + "github.com/grafana/pyroscope-go" + "k8s.io/klog/v2" +) + +const ( + PyroscopeServerEnvVar = "PYROSCOPE_SERVER" + PyroscopeAppNameEnvVar = "PYROSCOPE_APP_NAME" + PyroscopeAuthUserVar = "PYROSCOPE_AUTH_USER" + PyroscopeAuthPassVar = "PYROSCOPE_AUTH_PASSWORD" // #nosec G101 -- env var name, not a credential (nolint:gosec) + PyroscopeLogsEnabledEnvVar = "PYROSCOPE_LOGS_ENABLED" +) + +const defaultPyroscopeAppName = "porch-server" + +type PyroscopeProfiling struct { + stop func() error +} + +func (p *PyroscopeProfiling) Start() { + serverURL := os.Getenv(PyroscopeServerEnvVar) + if serverURL == "" { + return + } + + appName := os.Getenv(PyroscopeAppNameEnvVar) + if appName == "" { + appName = defaultPyroscopeAppName + } + + runtime.SetMutexProfileFraction(1) + runtime.SetBlockProfileRate(1) + + var logger pyroscope.Logger + logsEnabled := os.Getenv(PyroscopeLogsEnabledEnvVar) + if logsEnabled == "true" || logsEnabled == "1" { + logger = pyroscope.StandardLogger + } + + cfg := pyroscope.Config{ + ApplicationName: appName, + ServerAddress: serverURL, + Logger: logger, + Tags: map[string]string{ + "service_name": appName, + }, + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + } + if user := os.Getenv(PyroscopeAuthUserVar); user != "" { + cfg.BasicAuthUser = user + } + if password := os.Getenv(PyroscopeAuthPassVar); password != "" { + cfg.BasicAuthPassword = password + } + + profiler, err := pyroscope.Start(cfg) + if err != nil { + klog.Warningf("Failed to start Pyroscope profiling: %v", err) + return + } + p.stop = profiler.Stop + klog.Infof("Pyroscope continuous profiling started (server=%q, app=%q)", serverURL, appName) +} + +func (p *PyroscopeProfiling) Stop() { + if p.stop != nil { + klog.Infof("Stopping Pyroscope profiler") + if err := p.stop(); err != nil { + klog.Warningf("Pyroscope profiler stop: %v", err) + } + p.stop = nil + } +} diff --git a/pkg/apiserver/config.go b/pkg/apiserver/config.go index 5686a592d..97c83b855 100644 --- a/pkg/apiserver/config.go +++ b/pkg/apiserver/config.go @@ -49,6 +49,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -276,6 +277,9 @@ func (c *completedConfig) buildManager(restConfig *rest.Config, scheme *runtime. }, }, HealthProbeBindAddress: probePort, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, }) if err != nil { return nil, fmt.Errorf("error building manager: %w", err) diff --git a/pkg/externalrepo/git/git.go b/pkg/externalrepo/git/git.go index 185534567..000068046 100644 --- a/pkg/externalrepo/git/git.go +++ b/pkg/externalrepo/git/git.go @@ -37,6 +37,7 @@ import ( kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/errors" externalrepotypes "github.com/kptdev/porch/pkg/externalrepo/types" "github.com/kptdev/porch/pkg/repository" @@ -57,6 +58,8 @@ const ( fileName = "README.md" commitMessage = "Initial commit: Creating main branch" + telemetryName = "ExternalRepo" + // Retry delay constants baseRetryDelay = 200 * time.Millisecond hookRetryDelay = 1 * time.Second @@ -1129,6 +1132,9 @@ func (r *gitRepository) GetRepo() (string, error) { func (r *gitRepository) fetchRemoteRepository(ctx context.Context) error { ctx, span := tracer.Start(ctx, "gitRepository::fetchRemoteRepository", trace.WithAttributes()) defer span.End() + + telemetry.RecordRequestCount(ctx, telemetryName, "FETCH") + start := time.Now() defer func() { klog.V(2).Infof("Fetching repository %q took %s", r.key.Name, time.Since(start)) }() diff --git a/pkg/registry/porch/packagerevision.go b/pkg/registry/porch/packagerevision.go index ac42cb735..2d2a30e0f 100644 --- a/pkg/registry/porch/packagerevision.go +++ b/pkg/registry/porch/packagerevision.go @@ -17,8 +17,10 @@ package porch import ( "context" "fmt" + "time" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/repository" pctx "github.com/kptdev/porch/pkg/util/context" "go.opentelemetry.io/otel" @@ -33,6 +35,8 @@ import ( "k8s.io/klog/v2" ) +const prTelemetryName = "PackageRevision" + var tracer = otel.Tracer("packagerevision") type packageRevisions struct { @@ -72,7 +76,13 @@ func (r *packageRevisions) NamespaceScoped() bool { // List selects resources in the storage which match to the selector. 'options' can be nil. func (r *packageRevisions) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisions::List", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prTelemetryName, "LIST", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "LIST") ctx = pctx.WithNewRequestID(ctx) @@ -115,7 +125,13 @@ func (r *packageRevisions) List(ctx context.Context, options *metainternalversio // Get implements the Getter interface func (r *packageRevisions) Get(ctx context.Context, name string, _ *metav1.GetOptions) (runtime.Object, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisions::Get", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prTelemetryName, "GET", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "GET") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -141,7 +157,13 @@ func (r *packageRevisions) Get(ctx context.Context, name string, _ *metav1.GetOp func (r *packageRevisions) Create(ctx context.Context, runtimeObject runtime.Object, _ rest.ValidateObjectFunc, _ *metav1.CreateOptions) (runtime.Object, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisions::Create", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prTelemetryName, "CREATE", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "CREATE") ctx = pctx.WithNewRequestID(ctx) @@ -282,7 +304,13 @@ func createAction(pkgRev *porchapi.PackageRevision) string { func (r *packageRevisions) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, _ *metav1.UpdateOptions) (runtime.Object, bool, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisions::Update", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prTelemetryName, "UPDATE", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "UPDATE") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -306,7 +334,13 @@ func (r *packageRevisions) Update(ctx context.Context, name string, objInfo rest // deleted or false if it will be deleted asynchronously. func (r *packageRevisions) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, _ *metav1.DeleteOptions) (runtime.Object, bool, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisions::Delete", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prTelemetryName, "DELETE", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "DELETE") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) diff --git a/pkg/registry/porch/packagerevision_approval.go b/pkg/registry/porch/packagerevision_approval.go index f217aa425..231d8e80a 100644 --- a/pkg/registry/porch/packagerevision_approval.go +++ b/pkg/registry/porch/packagerevision_approval.go @@ -18,8 +18,10 @@ import ( "context" "fmt" "strings" + "time" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" pctx "github.com/kptdev/porch/pkg/util/context" "go.opentelemetry.io/otel/trace" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,6 +31,8 @@ import ( "k8s.io/klog/v2" ) +const praTelemetryName = "PackageRevisionApproval" + type packageRevisionApproval struct { packageCommon } @@ -53,7 +57,13 @@ func (a *packageRevisionApproval) NamespaceScoped() bool { func (a *packageRevisionApproval) Get(ctx context.Context, name string, _ *metav1.GetOptions) (runtime.Object, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisionApproval::Get", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(praTelemetryName, "GET", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "GET") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -70,7 +80,13 @@ func (a *packageRevisionApproval) Get(ctx context.Context, name string, _ *metav func (a *packageRevisionApproval) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, _ bool, _ *metav1.UpdateOptions) (runtime.Object, bool, error) { ctx, span := tracer.Start(ctx, "[START]::packageRevisionApproval::Update", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(praTelemetryName, "UPDATE", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prTelemetryName, "GET") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) diff --git a/pkg/registry/porch/packagerevisionresources.go b/pkg/registry/porch/packagerevisionresources.go index 15c9ff8c9..865ca7d7b 100644 --- a/pkg/registry/porch/packagerevisionresources.go +++ b/pkg/registry/porch/packagerevisionresources.go @@ -17,10 +17,12 @@ package porch import ( "context" "fmt" + "time" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/repository" pctx "github.com/kptdev/porch/pkg/util/context" "go.opentelemetry.io/otel/trace" @@ -35,6 +37,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const prrTelemetryName = "PackageRevisionResources" + type packageRevisionResources struct { rest.TableConvertor packageCommon @@ -69,7 +73,13 @@ func (r *packageRevisionResources) NamespaceScoped() bool { // List selects resources in the storage which match to the selector. 'options' can be nil. func (r *packageRevisionResources) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { ctx, span := tracer.Start(ctx, "[START]::PackageRevisionResources::List", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prrTelemetryName, "LIST", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prrTelemetryName, "LIST") ctx = pctx.WithNewRequestID(ctx) @@ -110,7 +120,13 @@ func (r *packageRevisionResources) List(ctx context.Context, options *metaintern // Get implements the Getter interface func (r *packageRevisionResources) Get(ctx context.Context, name string, _ *metav1.GetOptions) (runtime.Object, error) { ctx, span := tracer.Start(ctx, "[START]::PackageRevisionResources::Get", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prrTelemetryName, "GET", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prrTelemetryName, "GET") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -138,7 +154,13 @@ func (r *packageRevisionResources) Get(ctx context.Context, name string, _ *meta func (r *packageRevisionResources) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, _ rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, _ bool, _ *metav1.UpdateOptions) (runtime.Object, bool, error) { ctx, span := tracer.Start(ctx, "[START]::PackageRevisionResources::Update", trace.WithAttributes()) - defer span.End() + start := time.Now() + defer func() { + span.End() + telemetry.RecordAPICallDuration(prrTelemetryName, "UPDATE", time.Since(start).Seconds()) + }() + + telemetry.RecordRequestCount(ctx, prrTelemetryName, "UPDATE") ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) diff --git a/scripts/deploy-monitoring.sh b/scripts/deploy-monitoring.sh index 83287754a..a1b5d9e54 100755 --- a/scripts/deploy-monitoring.sh +++ b/scripts/deploy-monitoring.sh @@ -31,6 +31,8 @@ PROMETHEUS_NODEPORT="${PROMETHEUS_NODEPORT:-30091}" GRAFANA_LOCAL_PORT="${GRAFANA_LOCAL_PORT:-3001}" GRAFANA_CONTAINER_PORT="${GRAFANA_CONTAINER_PORT:-3000}" GRAFANA_NODEPORT="${GRAFANA_NODEPORT:-30301}" +PYROSCOPE_LOCAL_PORT="${PYROSCOPE_LOCAL_PORT:-4040}" +PYROSCOPE_CONTAINER_PORT="${PYROSCOPE_CONTAINER_PORT:-4040}" GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-porch}" GRAFANA_ADMIN_PW="${GRAFANA_ADMIN_PW:-}" @@ -42,6 +44,16 @@ PROMETHEUS_VERSION="${PROMETHEUS_VERSION:-latest}" PROMETHEUS_IMAGE="${DOCKERHUB_MIRROR}/prom/prometheus:${PROMETHEUS_VERSION}" GRAFANA_VERSION="${GRAFANA_VERSION:-latest}" GRAFANA_IMAGE="${DOCKERHUB_MIRROR}/grafana/grafana:${GRAFANA_VERSION}" +PYROSCOPE_VERSION="${PYROSCOPE_VERSION:-latest}" +PYROSCOPE_IMAGE="${DOCKERHUB_MIRROR}/grafana/pyroscope:${PYROSCOPE_VERSION}" +ALLOY_VERSION="${ALLOY_VERSION:-latest}" +ALLOY_IMAGE="${DOCKERHUB_MIRROR}/grafana/alloy:${ALLOY_VERSION}" +POSTGRES_EXPORTER_VERSION="${POSTGRES_EXPORTER_VERSION:-v0.16.0}" +POSTGRES_EXPORTER_IMAGE="${DOCKERHUB_MIRROR}/prometheuscommunity/postgres-exporter:${POSTGRES_EXPORTER_VERSION}" +POSTGRES_EXPORTER_CONTAINER_PORT="${POSTGRES_EXPORTER_CONTAINER_PORT:-9187}" +POSTGRES_DB_USER="${POSTGRES_DB_USER:-porch}" +POSTGRES_DB_PASSWORD="${POSTGRES_DB_PASSWORD:-porch}" +POSTGRES_EXPORTER_DATA_SOURCE_URI="${POSTGRES_EXPORTER_DATA_SOURCE_URI:-porch-postgresql.porch-system.svc.cluster.local:5432/porch?sslmode=disable}" RED='\033[0;31m' GREEN='\033[0;32m' @@ -89,12 +101,19 @@ pipeline: configMap: prometheus-image: "${PROMETHEUS_IMAGE}" grafana-image: "${GRAFANA_IMAGE}" + pyroscope-image: "${PYROSCOPE_IMAGE}" + alloy-image: "${ALLOY_IMAGE}" + postgres-exporter-image: "${POSTGRES_EXPORTER_IMAGE}" prometheus-container-port: "${PROMETHEUS_CONTAINER_PORT}" grafana-container-port: "${GRAFANA_CONTAINER_PORT}" + postgres-exporter-container-port: "${POSTGRES_EXPORTER_CONTAINER_PORT}" prometheus-nodeport: "${PROMETHEUS_NODEPORT}" grafana-nodeport: "${GRAFANA_NODEPORT}" grafana-user: "$(echo -n "$GRAFANA_ADMIN_USER" | base64)" grafana-pw: "$(echo -n "$GRAFANA_ADMIN_PW" | base64)" + postgres-exporter-db-user: "$(echo -n "$POSTGRES_DB_USER" | base64)" + postgres-exporter-db-password: "$(echo -n "$POSTGRES_DB_PASSWORD" | base64)" + postgres-exporter-data-source-uri: "${POSTGRES_EXPORTER_DATA_SOURCE_URI}" - image: ${KRM_FN_REGISTRY_URL}/set-namespace:v0.4.1 configMap: namespace: ${NAMESPACE} @@ -141,6 +160,11 @@ deploy_monitoring() { -n "$NAMESPACE" \ --dry-run=client -o yaml | kubectl apply -f - + kubectl create configmap alloy-config \ + --from-file=config.alloy="${SCRIPT_DIR}/../deployments/metrics-resources/alloy-config.alloy" \ + -n "$NAMESPACE" \ + --dry-run=client -o yaml | kubectl apply -f - + declare -a grafana_dashboards while read -r dashboard_file; do @@ -183,14 +207,18 @@ get_service_urls() { PROMETHEUS_PF_PID=$! kubectl port-forward -n "${NAMESPACE}" deployment/grafana "${GRAFANA_LOCAL_PORT}":"${GRAFANA_CONTAINER_PORT}" > /dev/null 2>&1 & GRAFANA_PF_PID=$! + kubectl port-forward -n "${NAMESPACE}" deployment/pyroscope "${PYROSCOPE_LOCAL_PORT}":"${PYROSCOPE_CONTAINER_PORT}" > /dev/null 2>&1 & + PYROSCOPE_PF_PID=$! sleep 2 echo "${PROMETHEUS_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-prometheus-pf.pid echo "${GRAFANA_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-grafana-pf.pid + echo "${PYROSCOPE_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-pyroscope-pf.pid PROMETHEUS_URL="http://localhost:${PROMETHEUS_LOCAL_PORT}" GRAFANA_URL="http://localhost:${GRAFANA_LOCAL_PORT}" + PYROSCOPE_URL="http://localhost:${PYROSCOPE_LOCAL_PORT}" echo "" log_info "==========================================" @@ -203,11 +231,17 @@ get_service_urls() { log_info " Username: ${GRAFANA_ADMIN_USER}" log_info " Password: ${GRAFANA_ADMIN_PW}" log_info " stored in: kubectl -n porch-monitoring get secrets --selector app=grafana -o yaml" + log_info " Pyroscope: ${PYROSCOPE_URL}" echo "" + log_info " - Alloy discovers annotated pods in the cluster via profiles.grafana.com/*" + log_info " - porch-server, porch-controllers, function-runner (port name: pprof)" + log_info "" log_info " - Prometheus is scraping metrics from:" log_info " - porch-server port 9464" log_info " - porch-controller port 9464" log_info " - function-runner port 9464" + log_info " - postgres-exporter port ${POSTGRES_EXPORTER_CONTAINER_PORT}" + log_info " - perf tests on host 172.17.0.1:9095 (when -enable-prometheus is set)" log_info "" echo "" log_info "To stop port forwarding, run:" @@ -223,13 +257,13 @@ cleanup() { if kubectl get namespace "$NAMESPACE" &> /dev/null; then log_info "Deleting resources in namespace $NAMESPACE..." - kubectl delete deployment prometheus grafana -n "$NAMESPACE" --ignore-not-found=true - kubectl delete service prometheus grafana -n "$NAMESPACE" --ignore-not-found=true - kubectl delete configmap prometheus-config grafana-dashboards grafana-dashboards-provider grafana-datasources -n "$NAMESPACE" --ignore-not-found=true - kubectl delete secret grafana-admin-creds -n "$NAMESPACE" --ignore-not-found=true - kubectl delete serviceaccount prometheus -n "$NAMESPACE" --ignore-not-found=true - kubectl delete clusterrole prometheus --ignore-not-found=true - kubectl delete clusterrolebinding prometheus --ignore-not-found=true + kubectl delete deployment prometheus grafana pyroscope alloy postgres-exporter -n "$NAMESPACE" --ignore-not-found=true + kubectl delete service prometheus grafana pyroscope postgres-exporter -n "$NAMESPACE" --ignore-not-found=true + kubectl delete configmap prometheus-config alloy-config grafana-dashboards grafana-dashboards-provider grafana-datasources -n "$NAMESPACE" --ignore-not-found=true + kubectl delete secret grafana-admin-creds postgres-exporter-db-creds -n "$NAMESPACE" --ignore-not-found=true + kubectl delete serviceaccount prometheus pyroscope alloy -n "$NAMESPACE" --ignore-not-found=true + kubectl delete clusterrole prometheus alloy --ignore-not-found=true + kubectl delete clusterrolebinding prometheus alloy --ignore-not-found=true log_info "Deleting namespace $NAMESPACE..." kubectl delete namespace "$NAMESPACE" --ignore-not-found=true @@ -250,6 +284,9 @@ main() { deploy_monitoring wait_for_deployment prometheus wait_for_deployment grafana + wait_for_deployment pyroscope + wait_for_deployment alloy + wait_for_deployment postgres-exporter get_service_urls ;; cleanup) @@ -270,6 +307,17 @@ main() { echo " NAMESPACE - Kubernetes namespace (default: porch-monitoring)" echo " PROMETHEUS_NODEPORT - Prometheus NodePort (default: 30091)" echo " GRAFANA_NODEPORT - Grafana NodePort (default: 30301)" + echo " DOCKERHUB_MIRROR - Registry mirror for images (default: docker.io)" + echo " PROMETHEUS_VERSION - Prometheus image version (default: latest)" + echo " GRAFANA_VERSION - Grafana image version (default: latest)" + echo " PYROSCOPE_VERSION - Pyroscope image version (default: latest)" + echo " PYROSCOPE_LOCAL_PORT - Pyroscope local port-forward port (default: 4040)" + echo " ALLOY_VERSION - Grafana Alloy image version (default: latest)" + echo " POSTGRES_EXPORTER_VERSION - Postgres exporter image version (default: v0.16.0)" + echo " POSTGRES_EXPORTER_CONTAINER_PORT - Postgres exporter port (default: 9187)" + echo " POSTGRES_DB_USER - Postgres DB user for exporter (default: porch)" + echo " POSTGRES_DB_PASSWORD - Postgres DB password for exporter (default: porch)" + echo " POSTGRES_EXPORTER_DATA_SOURCE_URI - Postgres connection URI for exporter" echo "" echo "Requirements:" echo " - kpt CLI (install from: https://kpt.dev/installation/)" diff --git a/test/e2e/performance/README.md b/test/e2e/performance/README.md deleted file mode 100644 index 14fa27c5a..000000000 --- a/test/e2e/performance/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Iterative Performance Test - -## Setup - -Run `prometheus_in_docker.sh create`, this will create a docker network so that the container can reach the metrics server, -then starts a container of Prometheus reachable at `localhost:9090`. - -## Running - -You can see all arguments with their default values by running `go test -v ./... -help=true` in this directory. - -The `PERF` environment variable must be set to `1` in order for the test to start. - -### Examples - -`PERF=1 go test -v ./... -repos 1 -iterations 10,20,30` - -run 3 iterations, creating 10, then 20, then 30 control PackageRevisions in 1 Repository - -`PERF=1 go test -v ./... -repos 3 -iterations 10,20,10,30` - -run 4 iterations, creating 30 (10x3), then 60 (20x3), then 30 (10x3), then 90 (30*3) PackageRevisions across 3 Repositories - -## Teardown - -Run `prometheus_in_docker.sh clean`, this will delete the container and the network. - -## Example queries for Prometheus - -### propose (k8s client update) latency averaged over 1s + actual PR count - -``` -rate(porch_operation_duration_ms_sum{operation="propose"}[1s])/rate(porch_operation_duration_ms_count{operation="propose"}[1s]) -or -porch_package_revisions_count -``` - -### all k8s client operation latency averaged over 1s for control packages + actual PR count - -``` -rate(porch_operation_duration_ms_sum{name=~"iterative-.*"}[1s])/rate(porch_operation_duration_ms_count{name=~"iterative-.*"}[1s]) -or -porch_package_revisions_count -``` diff --git a/test/e2e/performance/config.yml b/test/e2e/performance/config.yml deleted file mode 100644 index 4524402b4..000000000 --- a/test/e2e/performance/config.yml +++ /dev/null @@ -1,12 +0,0 @@ -global: - scrape_interval: 0s100ms - evaluation_interval: 0s100ms - -scrape_configs: - - job_name: 'porch_metrics' - static_configs: - - targets: ['host.docker.internal:2113'] - scrape_interval: 0s100ms - -rule_files: - - "./rules.yml" \ No newline at end of file diff --git a/test/e2e/performance/iterative_test.go b/test/e2e/performance/iterative_test.go deleted file mode 100644 index 4d950d4ec..000000000 --- a/test/e2e/performance/iterative_test.go +++ /dev/null @@ -1,378 +0,0 @@ -package performance - -import ( - "encoding/json" - "flag" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "testing" - "time" - - porchapi "github.com/kptdev/porch/api/porch/v1alpha1" - suiteutils "github.com/kptdev/porch/test/e2e/suiteutils" - "github.com/stretchr/testify/suite" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - controlRepoName = "iterative-control" - controlPackageName = "control-package" - testRepoName = "iterative-test" - testPackageName = "test-package" - - startingWS = "v1" -) - -var ( - repoCount = flag.Int("repos", 1, "Number of repositories to create") - iterations = flag.String("iterations", "10,25,50", "Number of control packages to create PER REPOSITORY for each iteration (e.g. \"10,25,50,100\")") - sampling = flag.Int("sampling", 5, "Number of measurements to take per iteration") - writeToFile = flag.Bool("write-to-file", false, "Write results to file") - outputDir = flag.String("output-dir", ".build/perf/iterative", "Where to write the results to if '-write-to-file' is set") - printToLog = flag.Bool("print-to-log", false, "Print results to log") - cooldown = flag.Duration("cooldown", 0, "Time to wait between iterations") -) - -type IterativeTest struct { - PerformanceSuite - - // entire test vars - metrics []FullMetricsData - iterations []int - sampling int - repos []string - cooldown time.Duration - - // per subtest vars - controlCount int - iterationIndex int -} - -func TestIterative(t *testing.T) { - if os.Getenv("PERF") != "1" { - t.Skip("PERF != 1") - } - test := &IterativeTest{} - test.UseGitea = true - suite.Run(t, test) -} - -func (t *IterativeTest) SetupSuite() { - t.parseFlags() // validate flags before starting the git server - t.PerformanceSuite.SetupSuite() - t.createRepos() - t.metrics = []FullMetricsData{} -} - -func (t *IterativeTest) parseFlags() { - flag.Parse() - its, err := toIntSlice(*iterations) - if err != nil { - t.Fatalf("failed to parse iterations: %v", err) - } - if len(its) == 0 { - t.Fatalf("Must specify at least one iteration (-iterations)") - } - t.iterations = its - - if *repoCount < 1 { - t.Fatalf("Repository count must be at least 1 (-repos)") - } - t.repos = []string{} - for i := range *repoCount { - t.repos = append(t.repos, fmt.Sprintf("%s-%d", controlRepoName, i+1)) - } - - t.sampling = *sampling - if t.sampling < 1 { - t.Fatalf("Test package count must be at least 1 (-sampling)") - } - - t.cooldown = *cooldown -} - -func toIntSlice(s string) ([]int, error) { - regex := regexp.MustCompile(`^\d+(,\d+)*$`) - if !regex.MatchString(s) { - return nil, fmt.Errorf("cannot parse %q as int slice", s) - } - var out []int - for _, v := range strings.Split(s, ",") { - i, err := strconv.Atoi(v) - if err != nil { - return nil, err - } - out = append(out, i) - } - return out, nil -} - -func (t *IterativeTest) createRepos() { - var wg sync.WaitGroup - for _, repo := range t.repos { - wg.Add(1) - go func() { - defer wg.Done() - t.RegisterGitRepositoryF(t.GetPorchTestRepoURL(), repo, repo, suiteutils.GiteaUser, suiteutils.GiteaPassword) - }() - } - t.RegisterGitRepositoryF(t.GetPorchTestRepoURL(), testRepoName, "iterativedir", suiteutils.GiteaUser, suiteutils.GiteaPassword) - wg.Wait() -} - -func (t *IterativeTest) SetupSubTest() { - times := make([]MyDuration, len(t.repos)) - var wg sync.WaitGroup - for i, repo := range t.repos { - wg.Add(1) - go func() { - defer wg.Done() - times[i] = Measure(func() { t.createPackageRevisions(t.controlCount, repo) }) - }() - } - wg.Wait() - var timeToCreateAllRevisions MyDuration = 0 - for _, v := range times { - timeToCreateAllRevisions += v - } - t.metrics = append(t.metrics, FullMetricsData{ - ControlRevisionCount: t.controlCount, - IterationIndex: t.iterationIndex, - CreateControlRevisionsTotal: timeToCreateAllRevisions, - CreateControlRevisionsAvg: AvgDuration(timeToCreateAllRevisions, t.controlCount), - }) -} - -func (t *IterativeTest) TearDownSubTest() { - timeToDeleteAllRevisions := Measure(func() { t.deletePackageRevisions() }) - t.metrics[len(t.metrics)-1].DeleteControlRevisionsTotal = timeToDeleteAllRevisions - t.metrics[len(t.metrics)-1].DeleteControlRevisionsAvg = AvgDuration(timeToDeleteAllRevisions, t.controlCount) -} - -func (t *IterativeTest) TestIterative() { - for i, n := range t.iterations { - t.controlCount = n - t.iterationIndex = i + 1 - subtestName := fmt.Sprintf("Iterative-%d-%d", t.iterationIndex, t.controlCount) - result := t.Run(subtestName, func() { - var iterationMetrics []IterationMetricsData - for range t.sampling { - currentMetrics := t.collectMetrics() - t.ensureTestPackagesDeleted() - iterationMetrics = append(iterationMetrics, *currentMetrics) - } - t.mergeMetrics(iterationMetrics) - }) - if !result { - t.Errorf("%s failed, stopping early", subtestName) - break - } - if *writeToFile { - t.writeResult(&t.metrics[len(t.metrics)-1]) - } - if i != len(t.iterations)-1 { - <-time.After(t.cooldown) - } - } - if *printToLog { - t.printResults() - } - if *writeToFile { - t.writeResults() - } -} - -func (t *IterativeTest) ensureTestPackagesDeleted() { - list := &porchapi.PackageRevisionList{} - t.ListE(list, client.InNamespace(t.Namespace), client.MatchingFields{"spec.packageName": testPackageName}) - for _, pr := range list.Items { - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed - t.UpdateApprovalL(&pr) - t.DeleteL(&pr) - } -} - -// mergeMetrics averages the gathered metrics of a single iteration, -// then appends these to the overall results -func (t *IterativeTest) mergeMetrics(iterationMetrics []IterationMetricsData) { - summed := iterationMetrics[0] - length := len(iterationMetrics) - for _, metric := range iterationMetrics[1:] { - summed.List += metric.List - summed.Create += metric.Create - summed.UpdateResources += metric.UpdateResources - summed.GetAfterResourceUpdate += metric.GetAfterResourceUpdate - summed.Propose += metric.Propose - summed.GetAfterPropose += metric.GetAfterPropose - summed.Approve += metric.Approve - summed.GetAfterPublish += metric.GetAfterPublish - summed.DeleteProposed += metric.DeleteProposed - summed.GetAfterProposeDelete += metric.GetAfterProposeDelete - summed.Delete += metric.Delete - } - summed.List = AvgDuration(summed.List, length) - summed.Create = AvgDuration(summed.Create, length) - summed.UpdateResources = AvgDuration(summed.UpdateResources, length) - summed.GetAfterResourceUpdate = AvgDuration(summed.GetAfterResourceUpdate, length) - summed.Propose = AvgDuration(summed.Propose, length) - summed.GetAfterPropose = AvgDuration(summed.GetAfterPropose, length) - summed.Approve = AvgDuration(summed.Approve, length) - summed.GetAfterPublish = AvgDuration(summed.GetAfterPublish, length) - summed.DeleteProposed = AvgDuration(summed.DeleteProposed, length) - summed.GetAfterProposeDelete = AvgDuration(summed.GetAfterProposeDelete, length) - summed.Delete = AvgDuration(summed.Delete, length) - t.metrics[len(t.metrics)-1].IterationMetricsData = summed -} - -// collectMetrics runs the inner part of the test, collecting the specified metrics -func (t *IterativeTest) collectMetrics() *IterationMetricsData { - output := &IterationMetricsData{} - var pr *porchapi.PackageRevision - - output.List = Measure(func() { t.ListF(&porchapi.PackageRevisionList{}) }) - - output.Create = Measure(func() { pr = t.CreatePackageDraftF(testRepoName, testPackageName, startingWS) }) - t.T().Cleanup(func() { t.DeleteL(pr) }) - - resources := &porchapi.PackageRevisionResources{} - t.GetF(client.ObjectKey{Namespace: t.Namespace, Name: pr.Name}, resources) - resources.Spec.Resources["README.md"] = "# updated readme" - output.UpdateResources = Measure(func() { t.UpdateF(resources) }) - - output.GetAfterResourceUpdate = Measure(func() { t.GetF(client.ObjectKey{Namespace: t.Namespace, Name: pr.Name}, pr) }) - - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleProposed - output.Propose = Measure(func() { t.UpdateF(pr) }) - - output.GetAfterPropose = Measure(func() { t.GetF(client.ObjectKey{Namespace: t.Namespace, Name: pr.Name}, pr) }) - - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished - output.Approve = Measure(func() { t.UpdateApprovalF(pr) }) - - output.GetAfterPublish = Measure(func() { t.GetF(client.ObjectKey{Namespace: t.Namespace, Name: pr.Name}, pr) }) - - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed - output.DeleteProposed = Measure(func() { t.UpdateApprovalF(pr) }) - - output.GetAfterProposeDelete = Measure(func() { t.GetF(client.ObjectKey{Namespace: t.Namespace, Name: pr.Name}, pr) }) - - output.Delete = Measure(func() { t.DeleteF(pr) }) - - return output -} - -// createPackageRevisions creates and publishes n number of control PackageRevisions. -// These will all be revisions of the same Package -func (t *IterativeTest) createPackageRevisions(n int, repoName string) { - pr := t.CreatePackageDraftF(repoName, controlPackageName, startingWS) - t.publishPackageRevision(pr) - - for i := 2; i <= n; i++ { - pr = t.createNewPackageRevisionFrom(pr, repoName, i) - t.publishPackageRevision(pr) - } -} - -// deletePackageRevisions deletes all PackageRevisions in the test cluster, -// including proposing deletion. -func (t *IterativeTest) deletePackageRevisions() { - var wg sync.WaitGroup - for _, repo := range append(t.repos, testRepoName) { - wg.Add(1) - go func() { - defer wg.Done() - prs := &porchapi.PackageRevisionList{} - t.ListE(prs, client.MatchingFields{"spec.repository": repo}) - for _, pr := range prs.Items { - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed - t.UpdateApprovalL(&pr) - t.DeleteL(&pr) - } - }() - } - wg.Wait() -} - -// createNewPackageRevisionFrom takes a PackageRevision and creates a new PackageRevision from it -func (t *IterativeTest) createNewPackageRevisionFrom(pr *porchapi.PackageRevision, repoName string, i int) *porchapi.PackageRevision { - newPr := &porchapi.PackageRevision{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: pr.Namespace, - }, - Spec: porchapi.PackageRevisionSpec{ - PackageName: pr.Spec.PackageName, - WorkspaceName: fmt.Sprintf("v%d", i), - RepositoryName: repoName, - Tasks: pr.Spec.Tasks, - Lifecycle: porchapi.PackageRevisionLifecycleDraft, - }, - } - - t.CreateF(newPr) - return newPr -} - -// publishPackageRevision proposes, then publishes an already existing draft PackageRevision -func (t *IterativeTest) publishPackageRevision(pr *porchapi.PackageRevision) { - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleProposed - t.UpdateF(pr) - - pr.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished - t.UpdateApprovalF(pr) -} - -// printResults prints all the gathered metrics to the log -func (t *IterativeTest) printResults() { - for _, metrics := range t.metrics { - t.Logf("results for %d revisions:\n%+v", metrics.ControlRevisionCount, metrics) - } -} - -// writeResult writes the result of one iteration to a json file -func (t *IterativeTest) writeResult(data *FullMetricsData) { - if err := t.ensureOutputDir(); err != nil { - return - } - filename := fmt.Sprintf("%d-%d.json", data.IterationIndex, data.ControlRevisionCount) - filepth := filepath.Join(*outputDir, filename) - t.write(filepth, data) -} - -// writeResults writes the results of all iteration to a combined json file -func (t *IterativeTest) writeResults() { - if err := t.ensureOutputDir(); err != nil { - return - } - filepth := filepath.Join(*outputDir, "full.json") - t.write(filepth, t.metrics) -} - -func (t *IterativeTest) write(filename string, data any) { - file, err := os.Create(filename) - if err != nil { - t.Logf("unable to create %s: %v", filename, err) - return - } - marshalled, err := json.MarshalIndent(data, "", " ") - if err != nil { - t.Logf("failed to marshal metrics: %v", err) - return - } - if _, err = file.Write(marshalled); err != nil { - t.Logf("failed to write to %s: %v", filename, err) - } -} - -func (t *IterativeTest) ensureOutputDir() error { - if err := os.MkdirAll(*outputDir, 0775); err != nil { - t.Logf("unable to create %s: %v", *outputDir, err) - return err - } - return nil -} diff --git a/test/e2e/performance/iterative_types.go b/test/e2e/performance/iterative_types.go deleted file mode 100644 index 0f86e1983..000000000 --- a/test/e2e/performance/iterative_types.go +++ /dev/null @@ -1,61 +0,0 @@ -package performance - -import ( - "fmt" - "time" -) - -// MyDuration is a wrapper around time.Duration for specifically handling time in (float) milliseconds -type MyDuration time.Duration - -func (d MyDuration) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("%.4f", d.Milliseconds())), nil -} - -func (d MyDuration) Milliseconds() float64 { - return float64(d) / 1_000_000 -} - -func Measure(f func()) MyDuration { - start := time.Now() - f() - return MyDuration(time.Since(start)) -} - -func AvgDuration(total MyDuration, n int) MyDuration { - return MyDuration(int64(total) / int64(n)) -} - -type IterationMetricsData struct { - List, - Create, - UpdateResources, - GetAfterResourceUpdate, - Propose, - GetAfterPropose, - Approve, - GetAfterPublish, - DeleteProposed, - GetAfterProposeDelete, - Delete MyDuration -} - -type FullMetricsData struct { - ControlRevisionCount int - IterationIndex int - - CreateControlRevisionsTotal, - CreateControlRevisionsAvg, - DeleteControlRevisionsTotal, - DeleteControlRevisionsAvg MyDuration - - IterationMetricsData `json:",inline"` - - //ServerMemoryBytes int64 - //ControllersMemoryBytes int64 - //FuncRunnerMemoryBytes int64 - // - //ServerAvgCPULoad float32 - //ControllersAvgCPULoad float32 - //FuncRunnerAvgCPULoad float32 -} diff --git a/test/e2e/performance/performance_suite.go b/test/e2e/performance/performance_suite.go deleted file mode 100644 index 031a1c245..000000000 --- a/test/e2e/performance/performance_suite.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2025-2026 The kpt Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package performance - -import ( - "flag" - "fmt" - "net/http" - "time" - - porchapi "github.com/kptdev/porch/api/porch/v1alpha1" - suiteutils "github.com/kptdev/porch/test/e2e/suiteutils" - "github.com/prometheus/client_golang/prometheus/promhttp" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - metricsPort = flag.Uint("port", 2113, "Port on which to expose metrics") -) - -type PerformanceSuite struct { - suiteutils.TestSuiteWithGit - - metricsServer *http.Server - metricsShutdown chan struct{} -} - -func (t *PerformanceSuite) SetupSuite() { - flag.Parse() - t.TestSuiteWithGit.SetupSuite() - t.metricsServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", *metricsPort)} - t.metricsShutdown = make(chan struct{}) - t.ServeMetrics() -} - -func (t *PerformanceSuite) TearDownSuite() { - t.ShutdownMetrics() -} - -func (t *PerformanceSuite) ServeMetrics() { - go func() { - t.Logf("Starting metrics server") - http.Handle("/metrics", promhttp.Handler()) - if err := t.metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - t.Fatalf("Error starting metrics server: %v", err) - } - t.Logf("Metrics server stopped") - t.metricsShutdown <- struct{}{} - }() -} - -// ShutdownMetrics tries to gracefully shut down the metrics server -func (t *PerformanceSuite) ShutdownMetrics() { - err := t.metricsServer.Shutdown(t.GetContext()) - if err != nil { - t.Logf("Error shutting down metrics server: %v", err) - } - select { - case <-t.metricsShutdown: - t.Logf("Metrics server shutdown complete") - case <-time.After(5 * time.Second): - t.Logf("Metrics server shutdown timed out") - } -} - -func (t *PerformanceSuite) incrementGuage(obj client.Object) { - if !t.T().Failed() { - switch KindOf(obj) { - case KindPackageRevision: - if obj.(*porchapi.PackageRevision).Spec.Revision != -1 { - packageRevisionGuage.Inc() - } - case KindRepository: - repositoryGauge.Inc() - } - } -} - -func (t *PerformanceSuite) decrementGuage(obj client.Object) { - if !t.T().Failed() { - switch KindOf(obj) { - case KindPackageRevision: - if obj.(*porchapi.PackageRevision).Spec.Revision != -1 { - packageRevisionGuage.Dec() - } - case KindRepository: - repositoryGauge.Dec() - } - } -} - -func (t *PerformanceSuite) GetE(key client.ObjectKey, obj client.Object) { - t.T().Helper() - MeasureAndRecord(OperationGet, obj, func() { t.TestSuiteWithGit.GetE(key, obj) }) -} - -func (t *PerformanceSuite) GetF(key client.ObjectKey, obj client.Object) { - t.T().Helper() - MeasureAndRecord(OperationGet, obj, func() { t.TestSuiteWithGit.GetF(key, obj) }) -} - -func (t *PerformanceSuite) ListE(list client.ObjectList, opts ...client.ListOption) { - t.T().Helper() - MeasureAndRecord(OperationList, list, func() { t.TestSuiteWithGit.ListE(list, opts...) }) -} - -func (t *PerformanceSuite) ListF(list client.ObjectList, opts ...client.ListOption) { - t.T().Helper() - MeasureAndRecord(OperationList, list, func() { t.TestSuiteWithGit.ListF(list, opts...) }) -} - -func (t *PerformanceSuite) CreateF(obj client.Object, opts ...client.CreateOption) { - t.T().Helper() - MeasureAndRecord(OperationCreate, obj, func() { t.TestSuiteWithGit.CreateF(obj, opts...) }) - - t.incrementGuage(obj) -} - -func (t *PerformanceSuite) CreateE(obj client.Object, opts ...client.CreateOption) { - t.T().Helper() - MeasureAndRecord(OperationCreate, obj, func() { t.TestSuiteWithGit.CreateE(obj, opts...) }) - - t.incrementGuage(obj) -} - -func (t *PerformanceSuite) DeleteF(obj client.Object, opts ...client.DeleteOption) { - t.T().Helper() - MeasureAndRecord(OperationDelete, obj, func() { t.TestSuiteWithGit.DeleteF(obj, opts...) }) - - t.decrementGuage(obj) -} - -func (t *PerformanceSuite) DeleteE(obj client.Object, opts ...client.DeleteOption) { - t.T().Helper() - MeasureAndRecord(OperationDelete, obj, func() { t.TestSuiteWithGit.DeleteE(obj, opts...) }) - - t.decrementGuage(obj) -} - -func (t *PerformanceSuite) DeleteL(obj client.Object, opts ...client.DeleteOption) { - t.T().Helper() - hadError := false - handler := func(format string, args ...any) { - hadError = true - t.Logf(format, args...) - } - MeasureAndRecord(OperationDelete, obj, func() { t.DeleteEH(obj, handler, opts...) }) - if !hadError { - t.decrementGuage(obj) - } -} - -func (t *PerformanceSuite) UpdateF(obj client.Object, opts ...client.UpdateOption) { - t.T().Helper() - MeasureAndRecord(getUpdateOperation(obj), obj, func() { t.TestSuiteWithGit.UpdateF(obj, opts...) }) -} - -func (t *PerformanceSuite) UpdateE(obj client.Object, opts ...client.UpdateOption) { - t.T().Helper() - MeasureAndRecord(getUpdateOperation(obj), obj, func() { t.TestSuiteWithGit.UpdateE(obj, opts...) }) -} - -func getUpdateOperation(obj client.Object) Operation { - if pr, ok := obj.(*porchapi.PackageRevision); ok && pr.Spec.Lifecycle == porchapi.PackageRevisionLifecycleProposed { - return OperationPropose - } - return OperationUpdate -} - -func (t *PerformanceSuite) PatchF(obj client.Object, patch client.Patch, opts ...client.PatchOption) { - t.T().Helper() - MeasureAndRecord(OperationPatch, obj, func() { t.TestSuiteWithGit.PatchF(obj, patch, opts...) }) -} - -func (t *PerformanceSuite) PatchE(obj client.Object, patch client.Patch, opts ...client.PatchOption) { - t.T().Helper() - MeasureAndRecord(OperationPatch, obj, func() { t.TestSuiteWithGit.PatchE(obj, patch, opts...) }) -} - -func (t *PerformanceSuite) UpdateApprovalL(pr *porchapi.PackageRevision) *porchapi.PackageRevision { - t.T().Helper() - var ret *porchapi.PackageRevision - MeasureAndRecord(getUpdateApprovalOperation(pr), pr, func() { ret = t.TestSuiteWithGit.UpdateApprovalL(pr) }) - return ret -} - -func (t *PerformanceSuite) UpdateApprovalF(pr *porchapi.PackageRevision) *porchapi.PackageRevision { - t.T().Helper() - var ret *porchapi.PackageRevision - MeasureAndRecord(getUpdateApprovalOperation(pr), pr, func() { ret = t.TestSuiteWithGit.UpdateApprovalL(pr) }) - return ret -} - -func getUpdateApprovalOperation(pr *porchapi.PackageRevision) Operation { - switch pr.Spec.Lifecycle { - case porchapi.PackageRevisionLifecyclePublished: - return OperationPublish - case porchapi.PackageRevisionLifecycleDeletionProposed: - return OperationProposeDelete - default: - return OperationUpdateApproval - } -} - -// copied, so the operation is recorded -func (t *PerformanceSuite) CreatePackageDraftF(repository, packageName, workspace string) *porchapi.PackageRevision { - t.T().Helper() - pr := t.CreatePackageSkeleton(repository, packageName, workspace) - pr.Spec.Tasks = []porchapi.Task{ - { - Type: porchapi.TaskTypeInit, - Init: &porchapi.PackageInitTaskSpec{}, - }, - } - t.CreateF(pr) - return pr -} diff --git a/test/e2e/performance/prometheus.go b/test/e2e/performance/prometheus.go deleted file mode 100644 index 8fd740cff..000000000 --- a/test/e2e/performance/prometheus.go +++ /dev/null @@ -1,84 +0,0 @@ -package performance - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -type Operation string - -const ( - OperationGet Operation = "get" - OperationCreate Operation = "create" - OperationUpdate Operation = "update" - OperationUpdateApproval Operation = "update-approval" - OperationPatch Operation = "patch" - OperationPropose Operation = "propose" - OperationPublish Operation = "publish" - OperationProposeDelete Operation = "propose-delete" - OperationDelete Operation = "delete" - OperationList Operation = "list" -) - -const ( - LabelKind = "kind" - LabelOperation = "operation" - LabelName = "name" -) - -var labels = []string{LabelKind, LabelOperation, LabelName} - -var ( - KindPackageRevision = "PackageRevision" - KindPackageRevisionResources = "PackageRevisionResources" - KindRepository = "Repository" -) - -var ( - // Operation duration metrics - operationDuration = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "porch_operation_duration_ms", - Help: "Duration of Porch operations in milliseconds", - // Buckets: []float64{math.Inf(1)}, - Buckets: []float64{10, 25, 50, 100, 250}, - }, - labels, - ) - - // Operation counter metrics - operationCounter = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "porch_operations_total", - Help: "Total number of Porch operations", - }, - labels, - ) - - // How many repositories there are - repositoryGauge = promauto.NewGauge( - prometheus.GaugeOpts{ - Name: "porch_repositories_count", - Help: "Total number of repositories present", - }, - ) - - // How many package revisions there are - packageRevisionGuage = promauto.NewGauge( - prometheus.GaugeOpts{ - Name: "porch_package_revisions_count", - Help: "Total number of package revisions present", - }, - ) -) - -// RecordMetric records both duration and count for an operation -func RecordMetric(kind string, operation Operation, name string, duration float64) { - operationDuration.WithLabelValues(kind, string(operation), name).Observe(duration) - operationCounter.WithLabelValues(kind, string(operation), name).Inc() -} - -func MeasureAndRecord(op Operation, obj any, fn func()) { - time := Measure(func() { fn() }) - RecordMetric(KindOf(obj), op, NameOf(obj), time.Milliseconds()) -} diff --git a/test/e2e/performance/prometheus_in_docker.sh b/test/e2e/performance/prometheus_in_docker.sh deleted file mode 100755 index a8959aa2b..000000000 --- a/test/e2e/performance/prometheus_in_docker.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR=$(dirname $(readlink -e ${BASH_SOURCE[0]})) - -function create() { - if [[ ! $(docker network ls | grep prometheus) ]]; then - docker network create prometheus - fi - - docker run \ - -d \ - --name prometheus \ - --network prometheus \ - --add-host host.docker.internal:host-gateway \ - -p 9090:9090 \ - -v $SCRIPT_DIR/config.yml:/etc/prometheus/prometheus.yml \ - -v $SCRIPT_DIR/rules.yml:/etc/prometheus/rules.yml \ - prom/prometheus:v3.2.1 -} - -function clean() { - docker container stop prometheus ||: - docker container rm prometheus ||: - docker network rm prometheus ||: -} - -if [[ $# -ne 1 ]]; then - echo "specify 'create' or 'clean'" - exit 1 -fi - -case $1 in - create) - create - ;; - clean) - clean - ;; - *) - echo "$1 is not a valid operation, specify 'create' or 'clean'" - exit 1 - ;; -esac diff --git a/test/e2e/performance/rules.yml b/test/e2e/performance/rules.yml deleted file mode 100644 index 44cb98066..000000000 --- a/test/e2e/performance/rules.yml +++ /dev/null @@ -1,47 +0,0 @@ -groups: - - name: porch_rules - interval: 10s - rules: - - record: porch:package_revisions_count - expr: porch_package_revisions_count - - # Average duration for create operations - - record: porch:create_operation_duration_seconds - expr: rate(porch_operation_duration_ms_sum{operation="create"}[1m]) / rate(porch_operation_duration_ms_count{operation="create"}[1m]) - labels: - operation: create - - # Average duration for propose operations - - record: porch:propose_operation_duration_seconds - expr: rate(porch_operation_duration_ms_sum{operation="propose"}[1m]) / rate(porch_operation_duration_ms_count{operation="propose"}[1m]) - labels: - operation: propose - - # Average duration for approve operations - - record: porch:approve_operation_duration_seconds - expr: rate(porch_operation_duration_ms_sum{operation="approve"}[1m]) / rate(porch_operation_duration_ms_count{operation="approve"}[1m]) - labels: - operation: approve - - - - name: porch_alerts - rules: - # Alert for high propose operation latency - - alert: HighProposeLatency - expr: rate(porch_operation_duration_ms_sum{operation="propose"}[1m]) / rate(porch_operation_duration_ms_count{operation="propose"}[1m]) > 500 - for: 10s - labels: - severity: warning - annotations: - summary: "High latency on propose operations" - description: "Propose operation latency is above 500ms for more than 10 seconds." - - # Alert for high create operation latency - - alert: HighCreateLatency - expr: rate(porch_operation_duration_ms_sum{operation="create"}[1m]) / rate(porch_operation_duration_ms_count{operation="create"}[1m]) > 500 - for: 10s - labels: - severity: warning - annotations: - summary: "High latency on create operations" - description: "Create operation latency is above 500ms for more than 10 seconds." \ No newline at end of file diff --git a/test/e2e/performance/utils.go b/test/e2e/performance/utils.go deleted file mode 100644 index 75485d12c..000000000 --- a/test/e2e/performance/utils.go +++ /dev/null @@ -1,43 +0,0 @@ -package performance - -import ( - "fmt" - "reflect" - - porchapi "github.com/kptdev/porch/api/porch/v1alpha1" - configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type hasName interface { - GetName() string -} - -func KindOf(o any) string { - if c, ok := o.(client.Object); ok { - if kind := c.GetObjectKind().GroupVersionKind().Kind; kind != "" { - return kind - } - } - // is regular type - if name := reflect.TypeOf(o).Name(); name != "" { - return name - } - // is pointer - return reflect.TypeOf(o).Elem().Name() -} - -func NameOf(o any) string { - switch c := o.(type) { - case *porchapi.PackageRevision: - return fmt.Sprintf("%s/%s", c.Spec.RepositoryName, c.Spec.PackageName) - case *porchapi.PackageRevisionResources: - return fmt.Sprintf("%s/%s", c.Spec.RepositoryName, c.Spec.PackageName) - case *configapi.Repository: - return c.Name - case hasName: - return c.GetName() - default: - return "" - } -} diff --git a/test/performance/README.md b/test/performance/README.md index 3a103700c..8feb617da 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -1,11 +1,9 @@ -Copyright 2024, 2026 The kpt Authors +Copyright 2024 The Nephio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,13 +16,14 @@ limitations under the License. ## Prerequisites - Docker - Kubernetes CLI (kubectl) +- Kpt CLI - Go development environment - Access to GitHub repositories ## 1. Set Up Development Environment -Run the following script to set up the development environment: +Run the following script from the porch directory to set up the development environment: ```bash -https://github.com/kptdev/porch/blob/main/scripts/setup-dev-env.sh +./scripts/setup-dev-env.sh ``` ## 2. Build and Deploy Porch @@ -34,103 +33,128 @@ make run-in-kind ``` Once deployment is complete, you should see the related pods running. -## 3. Create a Demo Namespace -Create a namespace for the demo setup: -```bash -kubectl create namespace porch-demo -``` +## 3. Deploy Monitoring Stack (Optional) -## 4. Define Gitea Repositories in Porch -Apply the repository configuration file: ```bash -kubectl apply -f examples/tutorials/starting-with-porch/porch-repositories.yaml +./scripts/deploy-monitoring.sh deploy ``` -Note: The Gitea credentials secret is automatically created when running the tests. The configuration can be found in `gitea-secret.yaml`. +This will: +- Create a `porch-monitoring` namespace +- Deploy Prometheus on port 9090 inside the cluster +- Deploy Grafana on port 3000 inside the cluster +- Configure Prometheus as a Grafana datasource +- Load the Porch performance dashboard +- Setup port-forwarding: Prometheus on localhost:9092, Grafana on localhost:3001 + +**Important for kind clusters**: Prometheus is configured to scrape metrics from the host machine via the Docker gateway IP (172.17.0.1). The performance test exposes metrics on port 9095 on the host, and Prometheus running inside the kind cluster accesses them via `172.17.0.1:9095`. + +### Access the Monitoring Tools -## 5. Verify Repository Setup -Check the status of repositories in the porch-demo namespace: +- **Prometheus**: http://localhost:9092 (via port-forward) +- **Grafana**: http://localhost:3001/dashboards + +### Cleanup Monitoring Stack ```bash -kubectl get repositories -n porch-demo +./deploy-monitoring.sh cleanup ``` +This will delete all monitoring resources and stop port forwarding. -## 6. Run Performance Tests -Navigate to the performance test directory: +### Restart Monitoring Stack ```bash -cd test/performance/ +./deploy-monitoring.sh restart ``` -Execute the performance test command: +## 4. Run Performance Tests +Navigate to the performance test directory: ```bash -E2E=1 go test -v ./... -repos=4 -packages=4 -timeout 60m +cd test/performance/ ``` -Parameters: -- `repos`: Number of repositories to test -- `packages`: Number of package revisions in each repository +There are two main performance tests available: -Once the tests begin, the logs can be observed in logs dir a sample is shared below. -``` -=== Iteration 0 Results === -Operation Duration Status --------------------------------------------------- -Create Gitea Repository 331ms Success -Create Porch Repository 7ms Success -Wait for Porch Repository Ready 2.008s Success - -=== Iteration 1 Results === -Operation Duration Status --------------------------------------------------- -Create PackageRevision 457ms Success -Update to Proposed 561ms Success -Update to Published 699ms Success -Delete PackageRevision 3.171s Success - -=== Iteration 1 Results === -Operation Duration Status --------------------------------------------------- -Delete Repository 5ms Success - -=== Consolidated Performance Test Results === -Operation Min Max Avg Total ------------------------------------------------------------------------- -Update to Published 699ms 699ms 699ms 699ms -Delete PackageRevision 3.171s 3.171s 3.171s 3.171s -Create Gitea Repository 331ms 331ms 331ms 331ms -Create Porch Repository 7ms 7ms 7ms 7ms -Wait for Porch Repository Ready 2.008s 2.008s 2.008s 2.008s -Create PackageRevision 457ms 457ms 457ms 457ms -Update to Proposed 561ms 561ms 561ms 561ms +### Run Load Test +Creates a preset amount of repositories, packages, and revisions to simulate load on the Porch system. +```bash +LOAD_TEST=1 go test -v ./... -timeout 1h ``` +Make sure to scale the timeout as needed and adjust test parameters as required. +Namely `-repos`, `-packages`, and `-revisions` to simulate the desired load and `-enable-prometheus` to enable visual metrics. -## 7. Verify Prometheus Targets -Check the Prometheus targets: -``` -http://localhost:9090/targets +### Maximum Package Revisions Test +Tests the maximum number of package revisions that can be handled by Porch in a single repository. +```bash +MAX_PR_TEST=1 go test -v ./... -timeout 1h ``` -You should be able to run PromQL queries here. +Make sure to scale the timeout as needed and adjust test parameters as required -> recommended is 72h. + +### Available Test Parameters: +- `-repos`: Number of repositories to test +- `-packages`: Number of packages per repository +- `-revisions`: Number of package revisions per package +- `-repo-parallelism`: Number of repositories to create in parallel +- `-package-parallelism`: Number of packages to create in parallel per repository +- `-enable-deletion`: Enable deletion of package revisions at the end of the test +- `-error-rate`: Maximum percentage of package revisions allowed to fail lifecycle transition +- `-enable-prometheus`: Enable Prometheus metrics server (default: false) -> **do not enable if monitoring stack is not deployed** +- `-metrics-log-prefix`: Prefix for the timestamped metrics log file +- `-results-file`: File name for test results +- `-detailed-log-file`: File name for detailed log +- `-repo-results-csv`: File name for repository results CSV +- `-operations-csv`: File name for operations details CSV +- `-deletion-csv`: File name for deletion operations CSV +- `-kptfile-path`: Path to the Kptfile +- `-package-resources-path`: Path to the package resources + +All results are stored in logs files located in the `test/performance/` and `test/performance/logs` directories. + +## 5. Sample Output -## 8. Check Metrics Output -Retrieve the captured metrics using: ```bash -curl -k http://localhost:2113/metrics +LOAD_TEST=1 go test -v ./test/performance -namespace=db-demo -repos=1 -packages=1 -revisions=3 -enable-prometheus=true -enable-deletion=true -timeout 1h ``` -This should return all the collected performance metrics from the test execution. -## Prometheus Setup - -The performance testing framework uses a Kubernetes-based Prometheus setup. The configuration is stored in `prometheus-manifests.yaml` and includes: - -- ConfigMap with Prometheus configuration -- Deployment for the Prometheus pod -- Service to expose Prometheus - -The Prometheus instance is automatically deployed in the `porch-demo` namespace when running the tests. You can access the Prometheus UI at: ``` -http://localhost:9090 +=== Consolidated Performance Test Results === +Operation Min Max Avg Total +------------------------------------------------------------------------------------ +Create Gitea Repository R0 272ms 272ms 272ms 272ms +Create Porch Repository R0 3ms 3ms 3ms 3ms +Repository Ready Wait R0 2.004s 2.004s 2.004s 2.004s +Package Revision List v1 9ms 9ms 9ms 9ms +Package Revision List v2 6ms 6ms 6ms 6ms +Package Revision List v3 6ms 6ms 6ms 6ms +Package Revision Create v1 10ms 10ms 10ms 10ms +Package Revision Create v2 9ms 9ms 9ms 9ms +Package Revision Create v3 9ms 9ms 9ms 9ms +Package Revision Render v1 4ms 4ms 4ms 4ms +Package Revision Render v2 14ms 14ms 14ms 14ms +Package Revision Render v3 13ms 13ms 13ms 13ms +Package Revision Get Resources v1 3ms 3ms 3ms 3ms +Package Revision Get Resources v2 3ms 3ms 3ms 3ms +Package Revision Get Resources v3 3ms 3ms 3ms 3ms +Package Revision Update v1 10ms 10ms 10ms 10ms +Package Revision Update v2 10ms 10ms 10ms 10ms +Package Revision Update v3 9ms 9ms 9ms 9ms +Package Revision Render v1 14ms 14ms 14ms 14ms +Package Revision Render v2 10ms 10ms 10ms 10ms +Package Revision Render v3 10ms 10ms 10ms 10ms +Package Revision Get v1 2ms 2ms 2ms 2ms +Package Revision Get v2 2ms 2ms 2ms 2ms +Package Revision Get v3 2ms 2ms 2ms 2ms +Package Revision Propose v1 11ms 11ms 11ms 11ms +Package Revision Propose v2 9ms 9ms 9ms 9ms +Package Revision Propose v3 10ms 10ms 10ms 10ms +Package Revision Get (Proposed) v1 2ms 2ms 2ms 2ms +Package Revision Get (Proposed) v2 2ms 2ms 2ms 2ms +Package Revision Get (Proposed) v3 3ms 3ms 3ms 3ms +Package Revision Approve/Publish v1 348ms 348ms 348ms 348ms +Package Revision Approve/Publish v2 339ms 339ms 339ms 339ms +Package Revision Approve/Publish v3 340ms 340ms 340ms 340ms +Package Revision Propose Deletion v1 8ms 8ms 8ms 8ms +Package Revision Propose Deletion v2 6ms 6ms 6ms 6ms +Package Revision Propose Deletion v3 9ms 9ms 9ms 9ms +Package Revision Delete v1 262ms 262ms 262ms 262ms +Package Revision Delete v2 262ms 262ms 262ms 262ms +Package Revision Delete v3 269ms 269ms 269ms 269ms ``` - -The configuration can be modified by editing `prometheus-manifests.yaml`. Key settings include: -- Scrape interval: 1s -- Resource limits: 512Mi memory, 200m CPU -- NodePort: 30090 \ No newline at end of file diff --git a/test/performance/csv_generation.go b/test/performance/csv_generation.go new file mode 100644 index 000000000..b0d549377 --- /dev/null +++ b/test/performance/csv_generation.go @@ -0,0 +1,346 @@ +// Copyright 2024-2025 The Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "encoding/csv" + "fmt" + "os" + "sort" + "time" + + pkgerrors "github.com/pkg/errors" +) + +func (t *PerfTestSuite) generateCSVResults() error { + csvFile, err := os.Create(t.csvOptions.lifecycleCSV) + if err != nil { + return pkgerrors.Wrapf(err, "failed to create CSV file %s", t.csvOptions.lifecycleCSV) + } + defer func() { _ = csvFile.Close() }() + + writer := csv.NewWriter(csvFile) + defer writer.Flush() + + header := []string{} + for i := 0; i < t.testOptions.numRepos; i++ { + repoSuffix := fmt.Sprintf("%0*d", len(fmt.Sprintf("%d", t.testOptions.numRepos)), i) + header = append(header, + fmt.Sprintf("REPO-%s-PKG:REV", repoSuffix), + fmt.Sprintf("REPO-%s-TOTAL-LIFECYCLE-DURATION", repoSuffix), + ) + } + + if err := writer.Write(header); err != nil { + return err + } + + type pkgRevResult struct { + pkgName string + revision int + totalDur time.Duration + } + + repoResults := make(map[string][]pkgRevResult) + + t.metricsMutex.RLock() + for repoName, testMetric := range t.metrics { + for pkgName, revisions := range testMetric.pkgRevMetrics { + for revNum, revMetrics := range revisions { + var totalLifecycleDur time.Duration + lifecycleComplete := false + + if publishedOp, ok := revMetrics.Metrics[pkgRevPublished]; ok && publishedOp.Error == nil { + lifecycleComplete = true + } + + if lifecycleComplete { + for opKey, opMetric := range revMetrics.Metrics { + if opMetric.Error != nil { + continue + } + if opKey != pkgRevProposeDeletion && opKey != pkgRevDelete { + totalLifecycleDur += opMetric.Duration + } + } + } + + repoResults[repoName] = append(repoResults[repoName], pkgRevResult{ + pkgName: pkgName, + revision: revNum, + totalDur: totalLifecycleDur, + }) + } + } + + sort.Slice(repoResults[repoName], func(i, j int) bool { + if repoResults[repoName][i].pkgName != repoResults[repoName][j].pkgName { + return repoResults[repoName][i].pkgName < repoResults[repoName][j].pkgName + } + return repoResults[repoName][i].revision < repoResults[repoName][j].revision + }) + } + t.metricsMutex.RUnlock() + + maxRows := 0 + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + if len(repoResults[repoName]) > maxRows { + maxRows = len(repoResults[repoName]) + } + } + + for row := 0; row < maxRows; row++ { + record := []string{} + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + results := repoResults[repoName] + + if row < len(results) { + result := results[row] + pkgRev := fmt.Sprintf("%s:v%d", result.pkgName, result.revision) + duration := formatDuration(result.totalDur) + record = append(record, pkgRev, duration) + } else { + record = append(record, "", "0") + } + } + + if err := writer.Write(record); err != nil { + return err + } + } + + return nil +} + +func (t *PerfTestSuite) generateDetailedOperationsCSV() error { + csvFile, err := os.Create(t.csvOptions.operationsCSV) + if err != nil { + return pkgerrors.Wrapf(err, "failed to create operations CSV file %s", t.csvOptions.operationsCSV) + } + defer func() { _ = csvFile.Close() }() + + writer := csv.NewWriter(csvFile) + defer writer.Flush() + + header := []string{} + for i := 0; i < t.testOptions.numRepos; i++ { + repoSuffix := fmt.Sprintf("%0*d", len(fmt.Sprintf("%d", t.testOptions.numRepos)), i) + header = append(header, + fmt.Sprintf("REPO-%s-PKG:REV", repoSuffix), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevList), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevCreate), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevResourcesGet), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevUpdate), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevGet), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevPropose), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevGetProposed), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevPublished), + ) + } + + if err := writer.Write(header); err != nil { + return err + } + + type pkgRevOps struct { + pkgName string + revision int + ops map[string]time.Duration + } + + repoOps := make(map[string][]pkgRevOps) + + t.metricsMutex.RLock() + for repoName, testMetric := range t.metrics { + for pkgName, revisions := range testMetric.pkgRevMetrics { + for revNum, revMetrics := range revisions { + ops := make(map[string]time.Duration) + + for opKey, opMetric := range revMetrics.Metrics { + if opMetric.Error != nil { + continue + } + + if opKey != pkgRevProposeDeletion && opKey != pkgRevDelete { + ops[opKey] = opMetric.Duration + } + } + + repoOps[repoName] = append(repoOps[repoName], pkgRevOps{ + pkgName: pkgName, + revision: revNum, + ops: ops, + }) + } + } + + sort.Slice(repoOps[repoName], func(i, j int) bool { + if repoOps[repoName][i].pkgName != repoOps[repoName][j].pkgName { + return repoOps[repoName][i].pkgName < repoOps[repoName][j].pkgName + } + return repoOps[repoName][i].revision < repoOps[repoName][j].revision + }) + } + t.metricsMutex.RUnlock() + + maxRows := 0 + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + if len(repoOps[repoName]) > maxRows { + maxRows = len(repoOps[repoName]) + } + } + + for row := 0; row < maxRows; row++ { + record := []string{} + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + ops := repoOps[repoName] + + if row < len(ops) { + op := ops[row] + pkgRev := fmt.Sprintf("%s:v%d", op.pkgName, op.revision) + record = append(record, pkgRev) + + record = append(record, + formatDuration(op.ops[pkgRevList]), + formatDuration(op.ops[pkgRevCreate]), + formatDuration(op.ops[pkgRevResourcesGet]), + formatDuration(op.ops[pkgRevUpdate]), + formatDuration(op.ops[pkgRevGet]), + formatDuration(op.ops[pkgRevPropose]), + formatDuration(op.ops[pkgRevGetProposed]), + formatDuration(op.ops[pkgRevPublished]), + ) + } else { + record = append(record, "", "0", "0", "0", "0", "0", "0", "0", "0") + } + } + + if err := writer.Write(record); err != nil { + return err + } + } + + return nil +} + +func (t *PerfTestSuite) generateDeletionOperationsCSV() error { + csvFile, err := os.Create(t.csvOptions.deletionCSV) + if err != nil { + return pkgerrors.Wrapf(err, "failed to create deletion CSV file %s", t.csvOptions.deletionCSV) + } + defer func() { _ = csvFile.Close() }() + + writer := csv.NewWriter(csvFile) + defer writer.Flush() + + header := []string{} + for i := 0; i < t.testOptions.numRepos; i++ { + repoSuffix := fmt.Sprintf("%0*d", len(fmt.Sprintf("%d", t.testOptions.numRepos)), i) + header = append(header, + fmt.Sprintf("REPO-%s-PKG:REV", repoSuffix), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevProposeDeletion), + fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevDelete), + ) + } + + if err := writer.Write(header); err != nil { + return err + } + + type pkgRevDelOps struct { + pkgName string + revision int + proposeDel time.Duration + deleteDur time.Duration + } + + repoDeletionOps := make(map[string][]pkgRevDelOps) + + t.metricsMutex.RLock() + for repoName, testMetric := range t.metrics { + for pkgName, revisions := range testMetric.pkgRevMetrics { + for revNum, revMetrics := range revisions { + var proposeDel, deleteDur time.Duration + + if proposeDelOp, ok := revMetrics.Metrics[pkgRevProposeDeletion]; ok && proposeDelOp.Error == nil { + proposeDel = proposeDelOp.Duration + } + if deleteOp, ok := revMetrics.Metrics[pkgRevDelete]; ok && deleteOp.Error == nil { + deleteDur = deleteOp.Duration + } + + if proposeDel > 0 || deleteDur > 0 { + repoDeletionOps[repoName] = append(repoDeletionOps[repoName], pkgRevDelOps{ + pkgName: pkgName, + revision: revNum, + proposeDel: proposeDel, + deleteDur: deleteDur, + }) + } + } + } + + sort.Slice(repoDeletionOps[repoName], func(i, j int) bool { + if repoDeletionOps[repoName][i].pkgName != repoDeletionOps[repoName][j].pkgName { + return repoDeletionOps[repoName][i].pkgName < repoDeletionOps[repoName][j].pkgName + } + return repoDeletionOps[repoName][i].revision < repoDeletionOps[repoName][j].revision + }) + } + t.metricsMutex.RUnlock() + + maxRows := 0 + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + if len(repoDeletionOps[repoName]) > maxRows { + maxRows = len(repoDeletionOps[repoName]) + } + } + + for row := 0; row < maxRows; row++ { + record := []string{} + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + ops := repoDeletionOps[repoName] + + if row < len(ops) { + op := ops[row] + pkgRev := fmt.Sprintf("%s:v%d", op.pkgName, op.revision) + proposeDelStr := formatDuration(op.proposeDel) + deleteStr := formatDuration(op.deleteDur) + record = append(record, pkgRev, proposeDelStr, deleteStr) + } else { + record = append(record, "", "0", "0") + } + } + + if err := writer.Write(record); err != nil { + return err + } + } + + return nil +} + +func formatDuration(d time.Duration) string { + if d == 0 { + return "0" + } + return fmt.Sprintf("%.3f", d.Seconds()) +} diff --git a/test/performance/logger.go b/test/performance/logger.go index b61f1f1a9..8b10b7323 100644 --- a/test/performance/logger.go +++ b/test/performance/logger.go @@ -1,4 +1,4 @@ -// Copyright 2024, 2026 The kpt Authors +// Copyright 2024-2025 The Nephio Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,49 +19,193 @@ import ( "log" "os" "path/filepath" - "testing" + "sync" "time" + + pkgerrors "github.com/pkg/errors" ) +const timeDateFormat = "2006-01-02 15:04:05" + type TestLogger struct { - t *testing.T file *os.File logger *log.Logger + closed bool + mutex sync.Mutex } -func NewTestLogger(t *testing.T) (*TestLogger, error) { - // Creates logs directory if it doesn't exist +func (t *PerfTestSuite) NewTestLogger(prefix string) (*TestLogger, error) { logsDir := "logs" if err := os.MkdirAll(logsDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create logs directory: %w", err) + return nil, pkgerrors.Wrapf(err, "failed to create logs directory %s", logsDir) } - // Create log file with timestamp - timestamp := time.Now().Format("2006-01-02-15-04-05") - filename := filepath.Join(logsDir, fmt.Sprintf("porch-metrics-%s.log", timestamp)) + timestamp := time.Now().Format(timeDateFormat) + filename := filepath.Join(logsDir, fmt.Sprintf("%s-%s.log", prefix, timestamp)) + + absPath, _ := filepath.Abs(filename) + fmt.Printf("Creating test log file: %s\n", absPath) + file, err := os.Create(filename) if err != nil { - return nil, fmt.Errorf("failed to create log file: %w", err) + return nil, pkgerrors.Wrapf(err, "failed to create log file %s", filename) } - logger := log.New(file, "", 0) // Remove timestamp from log entries + logger := log.New(file, "", 0) return &TestLogger{ - t: t, file: file, logger: logger, }, nil } +func (l *TestLogger) Sync() error { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.closed || l.file == nil { + return nil + } + return l.file.Sync() +} + func (l *TestLogger) Close() error { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.closed || l.file == nil { + return nil + } + + if err := l.file.Sync(); err != nil { + l.closed = true + _ = l.file.Close() + return err + } + + l.closed = true return l.file.Close() } -func (l *TestLogger) LogResult(format string, args ...any) { - if len(args) == 0 { - // If no args provided, treat format as plain string - l.logger.Println(format) - } else { - // Use as format string with args - l.logger.Printf(format, args...) +func (l *TestLogger) LogResult(format string, args ...interface{}) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.closed || l.file == nil { + return + } + + l.logger.Printf(format, args...) + _ = l.file.Sync() +} + +type ResultsLogger struct { + resultsFile *os.File + logFile *os.File + mutex sync.Mutex + resultsFileClosed bool + logFileClosed bool +} + +func (t *PerfTestSuite) NewResultsLogger(resultsFileName, logFileName string) (*ResultsLogger, error) { + resultsFile, err := os.Create(resultsFileName) + if err != nil { + return nil, pkgerrors.Wrapf(err, "failed to create results file %s", resultsFileName) + } + + logFile, err := os.Create(logFileName) + if err != nil { + _ = resultsFile.Close() + return nil, pkgerrors.Wrapf(err, "failed to create log file %s", logFileName) + } + + return &ResultsLogger{ + resultsFile: resultsFile, + logFile: logFile, + }, nil +} + +func (l *ResultsLogger) LogApproved(repoName, pkgName string, revision int, prName string, duration time.Duration) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.resultsFileClosed || l.resultsFile == nil { + return + } + + timestamp := time.Now().Format(timeDateFormat) + line := fmt.Sprintf("%s %s:%s:%d %s approved, took %.3f seconds\n", + timestamp, repoName, pkgName, revision, prName, duration.Seconds()) + _, _ = l.resultsFile.WriteString(line) + _ = l.resultsFile.Sync() +} + +func (l *ResultsLogger) LogDeleted(prName string, duration time.Duration) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.resultsFileClosed || l.resultsFile == nil { + return + } + + timestamp := time.Now().Format(timeDateFormat) + line := fmt.Sprintf("%s %s deleted, took %.3f seconds\n", + timestamp, prName, duration.Seconds()) + _, _ = l.resultsFile.WriteString(line) + _ = l.resultsFile.Sync() +} + +func (l *ResultsLogger) LogToFile(format string, args ...interface{}) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.logFileClosed || l.logFile == nil { + return + } + + _, _ = fmt.Fprintf(l.logFile, format+"\n", args...) + _ = l.logFile.Sync() +} + +func (l *ResultsLogger) Sync() error { + l.mutex.Lock() + defer l.mutex.Unlock() + + if !l.resultsFileClosed && l.resultsFile != nil { + if err := l.resultsFile.Sync(); err != nil { + return err + } + } + + if !l.logFileClosed && l.logFile != nil { + if err := l.logFile.Sync(); err != nil { + return err + } + } + + return nil +} + +func (l *ResultsLogger) Close() error { + l.mutex.Lock() + defer l.mutex.Unlock() + + var lastErr error + + if !l.resultsFileClosed && l.resultsFile != nil { + _ = l.resultsFile.Sync() + if err := l.resultsFile.Close(); err != nil { + lastErr = err + } + l.resultsFileClosed = true } + + if !l.logFileClosed && l.logFile != nil { + _ = l.logFile.Sync() + if err := l.logFile.Close(); err != nil { + lastErr = err + } + l.logFileClosed = true + } + + return lastErr } diff --git a/test/performance/metrics.go b/test/performance/metrics.go deleted file mode 100644 index aa0a6e63a..000000000 --- a/test/performance/metrics.go +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2024, 2026 The kpt Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package metrics - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "sync" - "testing" - "time" - - porchapi "github.com/kptdev/porch/api/porch/v1alpha1" - configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" -) - -var ( - scheme = runtime.NewScheme() -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(porchapi.AddToScheme(scheme)) - utilruntime.Must(configapi.AddToScheme(scheme)) -} - -const defaultGiteaLBIP = "172.18.255.200" - -var ( - giteaBaseURLOnce sync.Once - giteaBaseURLVal string -) - -// getGiteaBaseURL returns the Gitea base URL. It prefers the GITEA_LB_IP env var, -// then attempts to discover the IP from the gitea-lb Service in the cluster, -// and falls back to the hardcoded default only if both are unavailable. -func getGiteaBaseURL() string { - giteaBaseURLOnce.Do(func() { - ip := os.Getenv("GITEA_LB_IP") - if ip == "" { - ip = discoverGiteaLBIP() - } - if ip == "" { - ip = defaultGiteaLBIP - } - giteaBaseURLVal = "http://" + ip + ":3000" - }) - return giteaBaseURLVal -} - -// discoverGiteaLBIP attempts to read the gitea-lb Service LoadBalancer IP from the cluster. -func discoverGiteaLBIP() string { - cfg, err := config.GetConfig() - if err != nil { - return "" - } - c, err := client.New(cfg, client.Options{Scheme: scheme}) - if err != nil { - return "" - } - svc := &corev1.Service{} - err = c.Get(context.Background(), client.ObjectKey{Namespace: "gitea", Name: "gitea-lb"}, svc) - if err != nil { - return "" - } - if len(svc.Status.LoadBalancer.Ingress) > 0 { - return svc.Status.LoadBalancer.Ingress[0].IP - } - return "" -} - -func createGiteaRepo(repoName string) error { - giteaURL := getGiteaBaseURL() + "/api/v1/user/repos" - payload := map[string]any{ - "name": repoName, - "description": "Test repository for Porch metrics", - "private": false, - "auto_init": true, - } - - jsonPayload, err := json.Marshal(payload) - if err != nil { - return fmt.Errorf("failed to marshal payload: %w", err) - } - - req, err := http.NewRequest("POST", giteaURL, bytes.NewBuffer(jsonPayload)) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.SetBasicAuth("porch", "secret") - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to create repo: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusCreated { - return fmt.Errorf("failed to create repo, status: %d", resp.StatusCode) - } - - return nil -} - -func debugPackageStatus(t *testing.T, c client.Client, ctx context.Context, namespace, name string) { - var pkg porchapi.PackageRevision - err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &pkg) - if err != nil { - t.Logf("Error getting package: %v", err) - return - } - - t.Logf("\nPackage Status Details:") - t.Logf(" Name: %s", pkg.Name) - t.Logf(" LifecycleState: %s", pkg.Spec.Lifecycle) - t.Logf(" WorkspaceName: %s", pkg.Spec.WorkspaceName) - t.Logf(" Revision: %v", pkg.Spec.Revision) - t.Logf(" Published: %v", pkg.Status.PublishedAt) - t.Logf(" Tasks:") - for i, task := range pkg.Spec.Tasks { - t.Logf(" %d. Type: %s", i+1, task.Type) - if task.Type == porchapi.TaskTypeInit && task.Init != nil { - t.Logf(" Description: %s", task.Init.Description) - t.Logf(" Keywords: %v", task.Init.Keywords) - } - } - t.Logf(" Conditions:") - for _, cond := range pkg.Status.Conditions { - t.Logf(" - Type: %s", cond.Type) - t.Logf(" Status: %s", cond.Status) - t.Logf(" Message: %s", cond.Message) - t.Logf(" Reason: %s", cond.Reason) - } -} - -// ... existing imports and code ... - -func deleteGiteaRepo(repoName string) error { - giteaURL := fmt.Sprintf("%s/api/v1/repos/porch/%s", getGiteaBaseURL(), repoName) - - req, err := http.NewRequest("DELETE", giteaURL, nil) - if err != nil { - return fmt.Errorf("failed to create delete request: %w", err) - } - - req.SetBasicAuth("porch", "secret") - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to delete repo: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { - return fmt.Errorf("failed to delete repo, status: %d", resp.StatusCode) - } - - return nil -} -func waitForPorchRepository(ctx context.Context, c client.Client, t *testing.T, namespace, name string, timeout time.Duration) error { - start := time.Now() - for { - if time.Since(start) > timeout { - return fmt.Errorf("timeout waiting for repository to be ready") - } - - var repo configapi.Repository - err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &repo) - if err != nil { - return err - } - - t.Logf("\nRepository conditions at %v:", time.Since(start)) - t.Logf("Spec: %+v", repo.Spec) - t.Logf("Status: %+v", repo.Status) - - ready := false - for _, cond := range repo.Status.Conditions { - t.Logf(" - Type: %s, Status: %s, Message: %s", - cond.Type, cond.Status, cond.Message) - if cond.Type == "Ready" && cond.Status == "True" { - ready = true - break - } - } - - if ready { - return nil - } - - time.Sleep(2 * time.Second) - } -} diff --git a/test/performance/performance_suite.go b/test/performance/performance_suite.go new file mode 100644 index 000000000..e48368ea5 --- /dev/null +++ b/test/performance/performance_suite.go @@ -0,0 +1,995 @@ +// Copyright 2024-2025 The Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/joho/godotenv" + porchclient "github.com/kptdev/porch/api/generated/clientset/versioned" + porchapi "github.com/kptdev/porch/api/porch/v1alpha1" + configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" + pkgerrors "github.com/pkg/errors" + "github.com/stretchr/testify/suite" + coreapi "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +const prometheusPort = 9095 + +var ( + scheme = runtime.NewScheme() + + namespace = flag.String("namespace", "porch-metrics", "Kubernetes namespace to use for the test") + numRepos = flag.Int("repos", 1, "Number of repositories to create") + numPackages = flag.Int("packages", 5, "Number of packages per repository") + numRevisions = flag.Int("revisions", 5, "Number of package revisions per package") + repoParallelism = flag.Int("repo-parallelism", 1, "Number of repositories to create in parallel") + packageParallelism = flag.Int("package-parallelism", 1, "Number of packages to create in parallel per repository") + errorRate = flag.Float64("error-rate", 0.1, "Maximum percentage of package revisions allowed to fail lifecycle transition") + enableDeletion = flag.Bool("enable-deletion", false, "Enable deletion of package revisions at the end of the test") + enablePrometheus = flag.Bool("enable-prometheus", false, "Enable Prometheus metrics server on port 9091") + + metricsLogFile = flag.String("metrics-log-prefix", "porch-metrics", "Prefix for the timestamped metrics log file") + resultsFile = flag.String("results-file", "load_test_results.txt", "File name for test results") + fullLogFile = flag.String("detailed-log-file", "load_test.log", "File name for detailed log") + lifecycleCSV = flag.String("repo-results-csv", "load_test_lifecycle_results.csv", "File name for repository results CSV") + operationsCSV = flag.String("operations-csv", "load_test_operations_results.csv", "File name for operations details CSV") + deletionCSV = flag.String("deletion-csv", "load_test_deletion_results.csv", "File name for deletion operations CSV") + kptfilePath = flag.String("kptfile-path", "resources/Kptfile", "Path to the Kptfile") + packageResourcesPath = flag.String("package-resources-path", "resources/deployment.yaml", "Path to the package resources") + + giteaURL = flag.String("gitea-url", "http://localhost:3000", "Base URL for the Gitea API") + giteaUsername = flag.String("gitea-username", "porch", "Gitea username") + giteaPassword = flag.String("gitea-password", "secret", "Gitea password") + + paddingSize = flag.Int("padding-size", 0, "Size in MB to pad package resources with (0 disables padding)") + + retryBackoff = wait.Backoff{ + Duration: 50 * time.Millisecond, + Steps: 100, + Factor: 1.25, + Cap: 30 * time.Second, + } +) + +type PerfTestSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + client client.Client + clientSet porchclient.Interface + + testLogger *TestLogger + resultsLogger *ResultsLogger + otelResources *telemetry.OTelResources + enablePrometheus bool + + metrics map[string]TestMetrics + metricsMutex sync.RWMutex + + testOptions TestOptions + logOptions LogOptions + csvOptions CSVOptions +} + +type TestOptions struct { + namespace string + numRepos int + numPkgs int + numRevs int + repoParallelism int + packageParallelism int + errorRate float64 + enableDeletion bool + kptfilePath string + packageResourcesPath string + krmFnRegistryURL string + giteaURL string + giteaUsername string + giteaPassword string + paddingSize int +} + +type LogOptions struct { + metricsLogFile string + resultsFile string + fullLogFile string +} + +type CSVOptions struct { + lifecycleCSV string + operationsCSV string + deletionCSV string +} + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(porchapi.AddToScheme(scheme)) + utilruntime.Must(configapi.AddToScheme(scheme)) +} + +func (t *PerfTestSuite) recordRepoMetric(repoName, opKey string, op OperationMetrics) { + t.metricsMutex.Lock() + defer t.metricsMutex.Unlock() + t.metrics[repoName].repoOps[opKey] = op +} + +func (t *PerfTestSuite) recordPkgRevMetric(repoName, pkgName string, revisionNum int, opKey string, op OperationMetrics) { + t.metricsMutex.Lock() + defer t.metricsMutex.Unlock() + + repoMetrics, ok := t.metrics[repoName] + if !ok { + repoMetrics = TestMetrics{ + RepoName: repoName, + repoOps: make(map[string]OperationMetrics), + pkgRevMetrics: make(map[string]map[int]PackageRevisionMetrics), + } + } + if repoMetrics.pkgRevMetrics == nil { + repoMetrics.pkgRevMetrics = make(map[string]map[int]PackageRevisionMetrics) + } + if repoMetrics.pkgRevMetrics[pkgName] == nil { + repoMetrics.pkgRevMetrics[pkgName] = make(map[int]PackageRevisionMetrics) + } + + pkgRevEntry, exists := repoMetrics.pkgRevMetrics[pkgName][revisionNum] + if !exists || pkgRevEntry.Metrics == nil { + pkgRevEntry = PackageRevisionMetrics{ + pkgName: pkgName, + Revision: revisionNum, + Metrics: make(map[string]OperationMetrics), + } + } + pkgRevEntry.Metrics[opKey] = op + repoMetrics.pkgRevMetrics[pkgName][revisionNum] = pkgRevEntry + t.metrics[repoName] = repoMetrics +} + +func (t *PerfTestSuite) initPkgRevMetrics(repoName, pkgName string, revisionNum int) { + t.metricsMutex.Lock() + defer t.metricsMutex.Unlock() + t.metrics[repoName].pkgRevMetrics[pkgName][revisionNum] = PackageRevisionMetrics{ + pkgName: pkgName, + Revision: revisionNum, + Metrics: make(map[string]OperationMetrics), + } +} + +func getEnvWithDefault(key, defaultValue string) string { + _ = godotenv.Load(filepath.Join("..", "..", ".env")) + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +func (t *PerfTestSuite) SetupSuite() { + if os.Getenv("LOAD_TEST") != "1" && os.Getenv("MAX_PR_TEST=1") != "1" { + t.T().Skipf("Skipping performance tests in non-load test environment") + } + + flag.Parse() + + t.metrics = make(map[string]TestMetrics) + t.testOptions = TestOptions{ + namespace: *namespace, + numRepos: *numRepos, + numPkgs: *numPackages, + numRevs: *numRevisions, + repoParallelism: *repoParallelism, + packageParallelism: *packageParallelism, + errorRate: *errorRate, + enableDeletion: *enableDeletion, + kptfilePath: *kptfilePath, + packageResourcesPath: *packageResourcesPath, + krmFnRegistryURL: getEnvWithDefault("PORCH_GHCR_PREFIX_URL", "ghcr.io/kptdev/krm-functions-catalog"), + giteaURL: *giteaURL, + giteaUsername: *giteaUsername, + giteaPassword: *giteaPassword, + paddingSize: *paddingSize, + } + + t.logOptions = LogOptions{ + metricsLogFile: *metricsLogFile, + resultsFile: *resultsFile, + fullLogFile: *fullLogFile, + } + + t.csvOptions = CSVOptions{ + lifecycleCSV: *lifecycleCSV, + operationsCSV: *operationsCSV, + deletionCSV: *deletionCSV, + } + + logger, err := t.NewTestLogger(t.logOptions.metricsLogFile) + if err != nil { + t.T().Fatalf("Failed to create logger: %v", err) + } + + resultsLogger, err := t.NewResultsLogger(t.logOptions.resultsFile, t.logOptions.fullLogFile) + if err != nil { + t.T().Fatalf("Failed to create results logger: %v", err) + } + + cfg, err := config.GetConfig() + if err != nil { + t.T().Fatalf("Failed to get config: %v", err) + } + + c, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.T().Fatalf("Failed to create client: %v", err) + } + + clientSet, err := porchclient.NewForConfig(cfg) + if err != nil { + t.T().Fatalf("Failed to create Porch clientset: %v", err) + } + + t.ctx, t.cancel = context.WithCancel(context.Background()) + t.setupSignalHandler() + t.client = c + t.clientSet = clientSet + t.testLogger = logger + t.resultsLogger = resultsLogger + t.enablePrometheus = *enablePrometheus + + if t.enablePrometheus { + if err := os.Setenv("OTEL_METRICS_EXPORTER", "prometheus"); err != nil { + t.T().Fatalf("Failed to set OTEL_METRICS_EXPORTER: %v", err) + } + if err := os.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", strconv.Itoa(prometheusPort)); err != nil { + t.T().Fatalf("Failed to set OTEL_EXPORTER_PROMETHEUS_PORT: %v", err) + } + otelRes, err := telemetry.SetupOpenTelemetry(t.ctx) + if err != nil { + t.T().Fatalf("Failed to set up OpenTelemetry: %v", err) + } + t.otelResources = otelRes + t.T().Logf("OTel metrics server started on port %v", prometheusPort) + telemetry.PerfTestSetTestRunInfo("porch-performance-test", t.testOptions.namespace, time.Now()) + } + + t.T().Logf(" Running load test with:") + t.T().Logf(" Namespace: %s", t.testOptions.namespace) + t.T().Logf(" %d repositories", t.testOptions.numRepos) + t.T().Logf(" %d packages per repository", t.testOptions.numPkgs) + t.T().Logf(" %d revisions per package", t.testOptions.numRevs) + t.T().Logf(" Prometheus metrics: %v", t.enablePrometheus) + + if err = t.setupNamespaceAndSecret(); err != nil { + t.T().Fatalf("failed to setup namespace and secret: %v", err) + } + t.T().Logf("Created namespace %s and gitea secret", t.testOptions.namespace) + + t.T().Log("\n=== Cleaning up existing resources from previous runs ===") + if err = t.cleanupExistingResources(); err != nil { + t.T().Logf("Warning: Failed to cleanup existing resources: %v", err) + } + t.T().Log("Cleanup complete, ready to start test") +} + +func (t *PerfTestSuite) setupSignalHandler() { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + go func() { + sig := <-sigChan + signal.Stop(sigChan) + t.T().Logf("\nReceived signal %v, stopping test gracefully...", sig) + if t.cancel != nil { + t.cancel() + } + }() +} + +func (t *PerfTestSuite) TearDownSuite() { + interrupted := t.ctx.Err() != nil + if t.cancel != nil { + t.cancel() + } + if t.otelResources != nil { + if !interrupted { + t.T().Logf("Waiting 15 seconds before shutting down OTel resources to ensure final scrapes complete...") + time.Sleep(15 * time.Second) + } else { + t.T().Logf("Skipping OTel scrape wait due to test interruption") + } + + if err := t.otelResources.ShutdownWithTimeout(5 * time.Second); err != nil { + t.T().Logf("Warning: Failed to shut down OTel resources: %v", err) + } + } + if t.testLogger != nil { + if err := t.testLogger.Close(); err != nil { + t.T().Logf("Warning: Failed to close test logger: %v", err) + } + } + if t.resultsLogger != nil { + if err := t.resultsLogger.Close(); err != nil { + t.T().Logf("Warning: Failed to close results logger: %v", err) + } + } +} + +func (t *PerfTestSuite) setupNamespaceAndSecret() error { + ns := &coreapi.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.testOptions.namespace, + }, + } + + err := t.client.Create(t.ctx, ns) + if err != nil && !apierrors.IsAlreadyExists(err) { + return pkgerrors.Wrapf(err, "failed to create namespace %s", t.testOptions.namespace) + } + + secret := &coreapi.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gitea", + Namespace: t.testOptions.namespace, + }, + Type: coreapi.SecretTypeBasicAuth, + StringData: map[string]string{ + "username": t.testOptions.giteaUsername, + "password": t.testOptions.giteaPassword, + }, + } + + err = t.client.Create(t.ctx, secret) + if apierrors.IsAlreadyExists(err) { + existing := &coreapi.Secret{} + if err = t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: "gitea"}, existing); err != nil { + return pkgerrors.Wrapf(err, "failed to get existing gitea secret in namespace %s", t.testOptions.namespace) + } + existing.Type = coreapi.SecretTypeBasicAuth + existing.StringData = map[string]string{ + "username": t.testOptions.giteaUsername, + "password": t.testOptions.giteaPassword, + } + if err = t.client.Update(t.ctx, existing); err != nil { + return pkgerrors.Wrapf(err, "failed to update gitea secret in namespace %s", t.testOptions.namespace) + } + } else if err != nil { + return pkgerrors.Wrapf(err, "failed to create gitea secret in namespace %s", t.testOptions.namespace) + } + + return nil +} + +func (t *PerfTestSuite) cleanupExistingResources() error { + var repoList configapi.RepositoryList + if err := t.client.List(t.ctx, &repoList, client.InNamespace(t.testOptions.namespace)); err != nil { + if !apierrors.IsNotFound(err) { + return pkgerrors.Wrap(err, "failed to list repositories") + } + } else { + for _, repo := range repoList.Items { + if err := t.client.Delete(t.ctx, &repo); err != nil { + if !apierrors.IsNotFound(err) { + t.T().Errorf("failed to delete Repository %s: %v", repo.Name, err) + } + } + } + if len(repoList.Items) > 0 { + t.T().Logf("deleted %d existing Repositories", len(repoList.Items)) + time.Sleep(5 * time.Second) + } + } + + deletedCount := 0 + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + if err := deleteGiteaRepo(t.ctx, t.testOptions, repoName); err == nil { + deletedCount++ + } + } + if deletedCount > 0 { + t.T().Logf("deleted %d existing Gitea repositories", deletedCount) + } + + return nil +} + +func (t *PerfTestSuite) createAndSetupRepo(repoName string) { + t.metricsMutex.Lock() + t.metrics[repoName] = TestMetrics{ + RepoName: repoName, + repoOps: make(map[string]OperationMetrics), + pkgRevMetrics: make(map[string]map[int]PackageRevisionMetrics), + } + t.metricsMutex.Unlock() + + start := time.Now() + err := createGiteaRepo(t.ctx, t.testOptions, repoName) + duration := time.Since(start) + + t.recordRepoMetric(repoName, giteaRepoCreate, OperationMetrics{ + Operation: fmt.Sprintf("%s:%s", giteaRepoCreate, repoName), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(giteaRepoCreate, repoName, "", duration, err) + } + + if err != nil { + t.T().Errorf("Failed to create Gitea repository: %v", err) + return + } + + start = time.Now() + repo := &configapi.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: repoName, + Namespace: t.testOptions.namespace, + }, + Spec: configapi.RepositorySpec{ + Type: "git", + Git: &configapi.GitRepository{ + Repo: fmt.Sprintf("http://gitea.gitea.svc.cluster.local:3000/%s/%s", t.testOptions.giteaUsername, repoName), + Branch: "main", + SecretRef: configapi.SecretRef{ + Name: "gitea", + }, + CreateBranch: true, + }, + }, + } + + err = t.client.Create(t.ctx, repo) + duration = time.Since(start) + + t.recordRepoMetric(repoName, porchRepoCreate, OperationMetrics{ + Operation: fmt.Sprintf("%s:%s", porchRepoCreate, repoName), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(porchRepoCreate, repoName, "", duration, err) + } + + if err != nil { + t.T().Errorf("Failed to create Porch repository: %v", err) + return + } + + if t.enablePrometheus { + telemetry.PerfTestIncrementRepositoryCounter() + } + startWait := time.Now() + err = t.waitForRepository(repoName, 60*time.Second) + duration = time.Since(startWait) + + t.recordRepoMetric(repoName, repoWait, OperationMetrics{ + Operation: fmt.Sprintf("%s:%s", repoWait, repoName), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(repoWait, repoName, "", duration, err) + } +} + +func createGiteaRepo(ctx context.Context, opts TestOptions, repoName string) error { + giteaURL := fmt.Sprintf("%s/api/v1/user/repos", strings.TrimRight(opts.giteaURL, "/")) + payload := map[string]interface{}{ + "name": repoName, + "description": "Test repository for Porch metrics", + "private": false, + "auto_init": true, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return pkgerrors.Wrap(err, "failed to marshal payload") + } + + req, err := http.NewRequestWithContext(ctx, "POST", giteaURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return pkgerrors.Wrap(err, "failed to create request") + } + + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(opts.giteaUsername, opts.giteaPassword) + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return pkgerrors.Wrapf(err, "failed to create repo %s", repoName) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode != http.StatusCreated { + return pkgerrors.Errorf("failed to create repo, status: %d", resp.StatusCode) + } + + return nil +} + +func deleteGiteaRepo(ctx context.Context, opts TestOptions, repoName string) error { + url := fmt.Sprintf("%s/api/v1/repos/%s/%s", strings.TrimRight(opts.giteaURL, "/"), opts.giteaUsername, repoName) + + req, err := http.NewRequestWithContext(ctx, "DELETE", url, nil) + if err != nil { + return pkgerrors.Wrap(err, "failed to create delete request") + } + + req.SetBasicAuth(opts.giteaUsername, opts.giteaPassword) + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return pkgerrors.Wrapf(err, "failed to delete repo %s", repoName) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { + return pkgerrors.Errorf("failed to delete repo, status: %d", resp.StatusCode) + } + + return nil +} +func (t *PerfTestSuite) waitForRepository(name string, timeout time.Duration) error { + start := time.Now() + for { + if err := t.ctx.Err(); err != nil { + return err + } + if time.Since(start) > timeout { + return pkgerrors.Errorf("timeout waiting for repository to be ready") + } + + var repo configapi.Repository + err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: name}, &repo) + if err != nil { + return err + } + + t.T().Logf("\nRepository conditions at %v:", time.Since(start)) + t.T().Logf("Spec: %+v", repo.Spec) + t.T().Logf("Status: %+v", repo.Status) + + ready := false + for _, cond := range repo.Status.Conditions { + t.T().Logf(" - Type: %s, Status: %s, Message: %s", + cond.Type, cond.Status, cond.Message) + if cond.Type == "Ready" && cond.Status == "True" { + ready = true + break + } + } + + if ready { + return nil + } + + select { + case <-t.ctx.Done(): + return t.ctx.Err() + case <-time.After(2 * time.Second): + } + } +} + +func (t *PerfTestSuite) doLifecycle(repoName, pkgName string, revisionNum int) (string, error) { + var list porchapi.PackageRevisionList + var taskList []porchapi.Task + + t.initPkgRevMetrics(repoName, pkgName, revisionNum) + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.List(t.ctx, &list, client.InNamespace(t.testOptions.namespace)) + }) + duration := time.Since(start) + + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevList, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevList, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevList, repoName, pkgName, duration, err) + } + + if err != nil { + return "", err + } + + var latestPR *porchapi.PackageRevision + for i := range list.Items { + pr := &list.Items[i] + if pr.Spec.PackageName == pkgName && + pr.Spec.RepositoryName == repoName && + pr.Spec.Lifecycle == porchapi.PackageRevisionLifecyclePublished { + if latestPR == nil || pr.Spec.Revision > latestPR.Spec.Revision { + latestPR = pr + } + } + } + + if revisionNum == 1 { + taskList = []porchapi.Task{ + { + Type: porchapi.TaskTypeInit, + Init: &porchapi.PackageInitTaskSpec{ + Description: fmt.Sprintf("Test package %s for Porch metrics", pkgName), + Keywords: []string{"test", "metrics"}, + Site: "https://nephio.org", + }, + }, + } + if t.enablePrometheus { + telemetry.PerfTestIncrementPackageCounter() + } + } else if latestPR != nil { + taskList = []porchapi.Task{ + { + Type: porchapi.TaskTypeEdit, + Edit: &porchapi.PackageEditTaskSpec{ + Source: &porchapi.PackageRevisionRef{ + Name: latestPR.Name, + }, + }, + }, + } + } + + workspace := fmt.Sprintf("v%d", revisionNum) + pkgRev := &porchapi.PackageRevision{ + TypeMeta: metav1.TypeMeta{ + Kind: "PackageRevision", + APIVersion: porchapi.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.testOptions.namespace, + }, + Spec: porchapi.PackageRevisionSpec{ + PackageName: pkgName, + WorkspaceName: workspace, + RepositoryName: repoName, + Tasks: taskList, + }, + } + + if err = t.createPackageRevision(pkgRev, repoName, revisionNum); err != nil { + return "", err + } + + if err = t.updateOrCreatePackageRevisionResources(repoName, pkgName, pkgRev.Name, revisionNum); err != nil { + return "", err + } + + if err = t.proposeAndApprovePackage(repoName, pkgName, pkgRev.Name, revisionNum); err != nil { + return "", err + } + + return pkgRev.Name, nil +} + +func (t *PerfTestSuite) createPackageRevision(pkgRev *porchapi.PackageRevision, repoName string, revisionNum int) error { + start := time.Now() + if t.enablePrometheus { + telemetry.PerfTestRecordActiveOperation(pkgRevCreate, 1) + defer telemetry.PerfTestRecordActiveOperation(pkgRevCreate, -1) + } + + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Create(t.ctx, pkgRev) + }) + duration := time.Since(start) + + t.recordPkgRevMetric(repoName, pkgRev.Spec.PackageName, revisionNum, pkgRevCreate, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevCreate, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevCreate, repoName, pkgRev.Spec.PackageName, duration, err) + telemetry.PerfTestRecordPackageRevision(pkgRevCreate, err) + } + + if err != nil { + return err + } + + return nil +} + +func (t *PerfTestSuite) updateOrCreatePackageRevisionResources(repoName, pkgName, pkgRevName string, revisionNum int) error { + var resources porchapi.PackageRevisionResources + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &resources) + }) + duration := time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevResourcesGet, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevResourcesGet, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevResourcesGet, repoName, pkgName, duration, err) + } + + if err != nil { + return err + } + + pkgResources := t.createPackageResources(pkgRevName) + if resources.Spec.Resources == nil { + resources.Spec.Resources = make(map[string]string) + } + for name, content := range pkgResources { + resources.Spec.Resources[name] = content + } + + start = time.Now() + err = retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Update(t.ctx, &resources) + }) + duration = time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevUpdate, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevUpdate, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevUpdate, repoName, pkgName, duration, err) + } + + if err != nil { + return err + } + + return nil +} + +func (t *PerfTestSuite) createPackageResources(pkgName string) map[string]string { + resources := make(map[string]string) + + resources["Kptfile"] = t.readResourcesFromDir(t.testOptions.kptfilePath) + resources["deployment.yaml"] = t.readResourcesFromDir(t.testOptions.packageResourcesPath) + + resources["Kptfile"] = strings.ReplaceAll(resources["Kptfile"], "CHANGE_ME", pkgName) + resources["Kptfile"] = strings.ReplaceAll(resources["Kptfile"], "REGISTRY_URL", t.testOptions.krmFnRegistryURL) + resources["deployment.yaml"] = strings.ReplaceAll(resources["deployment.yaml"], "CHANGE_ME", pkgName) + + if t.testOptions.paddingSize > 0 { + resources["largefile.yaml"] = fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: padding-data +data: + value: "%s" +`, strings.Repeat("a", t.testOptions.paddingSize*1024*1024)) + } + + return resources +} + +func (t *PerfTestSuite) proposeAndApprovePackage(repoName, pkgName, pkgRevName string, revisionNum int) error { + var pkg porchapi.PackageRevision + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg) + }) + duration := time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevGet, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevGet, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevGet, repoName, pkgName, duration, err) + } + + if err != nil { + return err + } + + start = time.Now() + initialLifecycle := pkg.Spec.Lifecycle + err = retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg); err != nil { + return err + } + pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecycleProposed + return t.client.Update(t.ctx, &pkg) + }) + duration = time.Since(start) + + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevPropose, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevPropose, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevPropose, repoName, pkgName, duration, err) + telemetry.PerfTestRecordLifecycleTransition(string(initialLifecycle), string(porchapi.PackageRevisionLifecycleProposed), repoName, pkgName, duration, err) + } + + if err != nil { + return err + } + + start = time.Now() + err = retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg) + }) + duration = time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevGetProposed, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevGetProposed, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevGetProposed, repoName, pkgName, duration, err) + } + + if err != nil { + return err + } + + start = time.Now() + err = retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg); err != nil { + return err + } + pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished + _, err := t.clientSet.PorchV1alpha1().PackageRevisions(t.testOptions.namespace).UpdateApproval(t.ctx, pkgRevName, &pkg, metav1.UpdateOptions{}) + return err + }) + duration = time.Since(start) + + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevPublished, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevPublished, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevPublished, repoName, pkgName, duration, err) + telemetry.PerfTestRecordLifecycleTransition(string(porchapi.PackageRevisionLifecycleProposed), string(porchapi.PackageRevisionLifecyclePublished), repoName, pkgName, duration, err) + } + + return nil +} + +func (t *PerfTestSuite) deletePackageRevision(repoName, pkgName, pkgRevName string, revisionNum int) error { + var pkgRev porchapi.PackageRevision + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkgRev) + }) + if err != nil { + return err + } + + start := time.Now() + initialLifecycle := pkgRev.Spec.Lifecycle + err = retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkgRev); err != nil { + return err + } + pkgRev.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed + return t.client.Update(t.ctx, &pkgRev) + }) + duration := time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevProposeDeletion, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevProposeDeletion, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevProposeDeletion, repoName, pkgName, duration, err) + telemetry.PerfTestRecordLifecycleTransition(string(initialLifecycle), string(porchapi.PackageRevisionLifecycleDeletionProposed), repoName, pkgName, duration, err) + } + + if err != nil { + return err + } + + start = time.Now() + err = retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkgRev); err != nil { + return err + } + return t.client.Delete(t.ctx, &pkgRev) + }) + duration = time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevDelete, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevDelete, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + + if t.enablePrometheus { + telemetry.PerfTestRecordMetric(pkgRevDelete, repoName, pkgName, duration, err) + telemetry.PerfTestRecordLifecycleTransition(string(porchapi.PackageRevisionLifecycleDeletionProposed), "deleted", repoName, pkgName, duration, err) + } + + return nil +} + +func (t *PerfTestSuite) readResourcesFromDir(dir string) string { + t.T().Helper() + var content []byte + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + content, err = os.ReadFile(path) + if err != nil { + t.T().Fatalf("ReadFile(%q) failed: %v", path, err) + } + } + return nil + }) + if err != nil { + t.T().Fatalf("WalkDir(%s) failed: %v", dir, err) + } + return string(content) +} diff --git a/test/performance/porch_metrics_test.go b/test/performance/porch_metrics_test.go index 7f53022df..6b27ce1ed 100644 --- a/test/performance/porch_metrics_test.go +++ b/test/performance/porch_metrics_test.go @@ -1,4 +1,4 @@ -// Copyright 2024, 2026 The kpt Authors +// Copyright 2024-2025 The Nephio Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,540 +14,568 @@ package metrics import ( - "bytes" - "context" - "flag" "fmt" - "net/http" + "math" "os" - "os/exec" - "os/signal" "path/filepath" - "runtime" - "syscall" + "sort" + "strings" + "sync" "testing" "time" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" - configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" - "github.com/prometheus/client_golang/prometheus/promhttp" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/stretchr/testify/suite" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" ) -var ( - numRepos = flag.Int("repos", 1, "Number of repositories to create") - numPackages = flag.Int("packages", 5, "Number of packages per repository") -) - -func createAndSetupRepo(t *testing.T, ctx context.Context, c client.Client, namespace, repoName string) []OperationMetrics { - var metrics []OperationMetrics - start := time.Now() - - // Create Gitea repo - err := createGiteaRepo(repoName) - duration := time.Since(start).Seconds() - recordMetric("Create Gitea Repository", repoName, "", duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Create Gitea Repository", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) - - if err != nil { - t.Logf("Warning: Failed to create Gitea repository: %v", err) - return metrics - } - - // Create Porch repo - start = time.Now() - repo := &configapi.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: repoName, - Namespace: namespace, - }, - Spec: configapi.RepositorySpec{ - Type: "git", - Git: &configapi.GitRepository{ - Repo: fmt.Sprintf("%s/porch/%s", getGiteaBaseURL(), repoName), - Branch: "main", - SecretRef: configapi.SecretRef{ - Name: "gitea", - }, - CreateBranch: true, - }, - }, - } - - err = c.Create(ctx, repo) - duration = time.Since(start).Seconds() - recordMetric("Create Porch Repository", repoName, "", duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Create Porch Repository", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) - - if err == nil { - repositoryCounter.Inc() - start = time.Now() - err = waitForPorchRepository(ctx, c, t, namespace, repoName, 60*time.Second) - duration = time.Since(start).Seconds() - recordMetric("Wait Repository Ready", repoName, "", duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Wait for Porch Repository Ready", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) - } - - return metrics +type PerformanceTests struct { + PerfTestSuite } -func createAndTestPackage(t *testing.T, ctx context.Context, c client.Client, namespace, repoName, pkgName string) []OperationMetrics { - var metrics []OperationMetrics - start := time.Now() - - // Create new package - newPkg := &porchapi.PackageRevision{ - TypeMeta: metav1.TypeMeta{ - Kind: "PackageRevision", - APIVersion: porchapi.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("%s-", repoName), - Namespace: namespace, - }, - Spec: porchapi.PackageRevisionSpec{ - PackageName: pkgName, - WorkspaceName: "main", - RepositoryName: repoName, - Lifecycle: porchapi.PackageRevisionLifecycleDraft, - Tasks: []porchapi.Task{ - { - Type: porchapi.TaskTypeInit, - Init: &porchapi.PackageInitTaskSpec{ - Description: "Test package for Porch metrics", - Keywords: []string{"test", "metrics"}, - Site: "https://kpt.dev", - }, - }, - }, - }, - } - - err := c.Create(ctx, newPkg) - duration := time.Since(start).Seconds() - recordMetric("Create PackageRevision", repoName, pkgName, duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Create PackageRevision", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) - - if err == nil { - packageCounter.Inc() - } - - // Wait for package to initialize - time.Sleep(5 * time.Second) - debugPackageStatus(t, c, ctx, namespace, newPkg.Name) - - // First get the package - var pkg porchapi.PackageRevision - err = c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: newPkg.Name}, &pkg) - if err == nil { - // Start timing only the update operation - start = time.Now() - pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecycleProposed - err = c.Update(ctx, &pkg) - duration = time.Since(start).Seconds() - recordMetric("Update to Proposed", repoName, pkgName, duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Update to Proposed", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) - +func TestPerf(t *testing.T) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, err := os.UserHomeDir() if err == nil { - // Wait for proposed state to settle - time.Sleep(5 * time.Second) - debugPackageStatus(t, c, ctx, namespace, pkg.Name) - - // Publish the package with approval - start = time.Now() - pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished - err = c.SubResource("approval").Update(ctx, &pkg) - duration = time.Since(start).Seconds() - recordMetric("Update to Published", repoName, pkgName, duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Update to Published", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) - - if err == nil { - // Verify final state - time.Sleep(5 * time.Second) - debugPackageStatus(t, c, ctx, namespace, pkg.Name) - } + kubeconfig = filepath.Join(home, ".kube", "config") } } - // Delete package - err = c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: pkg.Name}, &pkg) - if err == nil { - start = time.Now() - pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed - err = c.SubResource("approval").Update(ctx, &pkg) - if err == nil { - time.Sleep(2 * time.Second) - err = c.Delete(ctx, &pkg) - } - duration = time.Since(start).Seconds() - recordMetric("Delete PackageRevision", repoName, pkgName, duration, err) - metrics = append(metrics, OperationMetrics{ - Operation: "Delete PackageRevision", - Duration: time.Duration(duration * float64(time.Second)), - Error: err, - }) + if _, err := os.Stat(kubeconfig); err == nil { + _ = os.Setenv("KUBERNETES_MASTER", "http://localhost:8080") } - return metrics + suite.Run(t, &PerformanceTests{}) } -func setupMonitoring(t *testing.T) error { - // Create prometheus.yml - promConfig := ` -global: - scrape_interval: 1s - evaluation_interval: 1s - -scrape_configs: - - job_name: 'porch_metrics' - static_configs: - - targets: ['host.docker.internal:2113'] - scrape_interval: 1s -` - if err := os.WriteFile("prometheus.yml", []byte(promConfig), 0644); err != nil { - return fmt.Errorf("failed to create prometheus.yml: %w", err) - } - - // Execute Docker commands - cmds := []struct { - name string - cmd string - args []string - }{ - {"network create", "docker", []string{"network", "create", "monitoring"}}, - {"stop prometheus", "docker", []string{"stop", "prometheus"}}, - {"remove prometheus", "docker", []string{"rm", "prometheus"}}, - {"run prometheus", "docker", []string{ - "run", "-d", - "--name", "prometheus", - "--network", "monitoring", - "--add-host", "host.docker.internal:host-gateway", - "-p", "9090:9090", - "-v", fmt.Sprintf("%s/prometheus.yml:/etc/prometheus/prometheus.yml", getCurrentDir()), - "prom/prometheus", - }}, - } - - for _, cmd := range cmds { - if err := exec.Command(cmd.cmd, cmd.args...).Run(); err != nil { - t.Logf("Warning executing %s: %v", cmd.name, err) - } +func (t *PerformanceTests) TestPorchScalePerformance() { + if os.Getenv("LOAD_TEST") != "1" { + t.T().Skipf("LOAD_TEST != 1: Skipping performance tests in non-load test environment") } - // Give Prometheus a moment to start up - time.Sleep(2 * time.Second) - - return nil -} - -func getCurrentDir() string { - dir, err := os.Getwd() - if err != nil { - return "." + // We never use error calculation in scale performance test + errorCalculator := func(err error, errCount, numRevs int) bool { + return false } - return dir -} -func cleanup(t *testing.T) { - cmds := []struct { - cmd string - args []string - }{ - {"docker", []string{"stop", "prometheus"}}, - {"docker", []string{"rm", "prometheus"}}, - {"docker", []string{"network", "rm", "monitoring"}}, - } + testStartTime := time.Now() - for _, cmd := range cmds { - if err := exec.Command(cmd.cmd, cmd.args...).Run(); err != nil { - t.Logf("Warning during cleanup: %v", err) + repoSemaphore := make(chan struct{}, t.testOptions.repoParallelism) + var repoWg sync.WaitGroup + + for i := 0; i < t.testOptions.numRepos; i++ { + if err := t.ctx.Err(); err != nil { + break } + repoSemaphore <- struct{}{} + repoWg.Add(1) + go func(repoIndex int) { + defer repoWg.Done() + defer func() { <-repoSemaphore }() + t.processRepository(repoIndex, t.testOptions.numRevs, errorCalculator) + }(i) } - - os.Remove("prometheus.yml") -} - -func setupGiteaSecret(t *testing.T) error { - // Get the current file's directory - _, filename, _, ok := runtime.Caller(0) - if !ok { - return fmt.Errorf("failed to get current file path") + repoWg.Wait() + lifecycleDuration := time.Since(testStartTime) + if err := t.ctx.Err(); err != nil { + t.T().Logf("Test interrupted: %v", err) } - dir := filepath.Dir(filename) - // Read and apply the secret manifest - secretManifest, err := os.ReadFile(filepath.Join(dir, "gitea-secret.yaml")) - if err != nil { - return fmt.Errorf("failed to read gitea secret manifest: %w", err) - } + var deletionStartTime time.Time + var deletionDuration time.Duration + var deletedCount int - cmd := exec.Command("kubectl", "apply", "-f", "-") - cmd.Stdin = bytes.NewReader(secretManifest) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to apply gitea secret: %w\nOutput: %s", err, output) + if t.testOptions.enableDeletion && t.ctx.Err() == nil { + t.deleteEnv(&deletionStartTime, &deletionDuration, &deletedCount) } - return nil -} + t.printTestResults(t.testLogger) -func TestPorchScalePerformance(t *testing.T) { - // Skip if not running E2E tests - if os.Getenv("E2E") == "" { - t.Skip("Skipping performance tests in non-E2E environment") + if t.testOptions.enableDeletion { + t.T().Logf("Total duration for deletion operations: %v", deletionDuration) + t.resultsLogger.LogToFile("Total duration for deletion operations: %v", deletionDuration) } + t.logResults(lifecycleDuration, &deletedCount) +} - // Check if docker is available - if _, err := exec.LookPath("docker"); err != nil { - t.Skip("Docker not available, skipping performance tests") +func (t *PerformanceTests) TestIncreasePRsPerformance() { + maxPkgRevNum := math.MaxInt + if os.Getenv("MAX_PR_TEST") != "1" { + t.T().Skipf("MAX_PR_TEST != 1: Skipping performance tests in non-load test environment") } - // Setup Gitea secret - if err := setupGiteaSecret(t); err != nil { - t.Fatalf("Failed to setup Gitea secret: %v", err) + // TODO: Making more complex error calculation logic + errorCalculator := func(err error, errCount, numRevs int) bool { + if err != nil { + t.T().Logf("\n--- Error Rate: %f", float64(errCount)/float64(numRevs)) + return float64(errCount)/float64(numRevs) >= t.testOptions.errorRate + } + return false } - // Setup monitoring - if err := setupMonitoring(t); err != nil { - t.Fatalf("Failed to setup monitoring: %v", err) - } - defer cleanup(t) + testStartTime := time.Now() - // Create a channel to handle interrupt signal - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + repoIndex := 0 - // Start metrics server - go func() { - http.Handle("/metrics", promhttp.Handler()) - if err := http.ListenAndServe(":2113", nil); err != nil { - t.Logf("Error starting metrics server: %v", err) - } - }() + t.processRepository(repoIndex, maxPkgRevNum, errorCalculator) - // Setup logger - logger, err := NewTestLogger(t) - if err != nil { - t.Fatalf("Failed to create logger: %v", err) + lifecycleDuration := time.Since(testStartTime) + if err := t.ctx.Err(); err != nil { + t.T().Logf("Test interrupted: %v", err) } - defer logger.Close() - flag.Parse() + var deletionStartTime time.Time + var deletionDuration time.Duration + var deletedCount int - t.Logf("\nRunning test with %d repositories and %d packages per repository", *numRepos, *numPackages) - - // Setup clients - cfg, err := config.GetConfig() - if err != nil { - t.Fatalf("Failed to get config: %v", err) + if t.testOptions.enableDeletion && t.ctx.Err() == nil { + t.deleteEnv(&deletionStartTime, &deletionDuration, &deletedCount) } - c, err := client.New(cfg, client.Options{Scheme: scheme}) - if err != nil { - t.Fatalf("Failed to create client: %v", err) + t.printTestResults(t.testLogger) + if t.testOptions.enableDeletion { + t.T().Logf("Total duration for deletion operations: %v", deletionDuration) + t.resultsLogger.LogToFile("Total duration for deletion operations: %v", deletionDuration) } + t.logResults(lifecycleDuration, &deletedCount) +} - ctx := context.Background() - namespace := "porch-demo" - var allMetrics []TestMetrics - - // Test multiple repositories - for i := 0; i < *numRepos; i++ { - repoName := fmt.Sprintf("porch-metrics-test-%d", i) - t.Logf("\n=== Testing Repository %d: %s ===", i+1, repoName) +func (t *PerformanceTests) deleteEnv(deletionStartTime *time.Time, deletionDuration *time.Duration, deletedCount *int) { - // Cleanup any existing resources first - _ = deleteGiteaRepo(repoName) - _ = c.Delete(ctx, &configapi.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: repoName, - Namespace: namespace, - }, - }) - time.Sleep(5 * time.Second) // Wait for cleanup + *deletionStartTime = time.Now() - repoMetrics := createAndSetupRepo(t, ctx, c, namespace, repoName) - for _, m := range repoMetrics { - recordMetric(m.Operation, repoName, "", m.Duration.Seconds(), m.Error) - } - printIterationResults(t, logger, i*(*numPackages), repoMetrics) + t.T().Log("\n=== Starting Deletion Operations ===") + t.T().Logf("Deletion enabled: will delete all %d package revisions across %d repositories", t.testOptions.numRepos*t.testOptions.numPkgs*t.testOptions.numRevs, t.testOptions.numRepos) - // Test multiple packages per repository - for j := 0; j < *numPackages; j++ { - pkgName := fmt.Sprintf("test-package-%d", j) - t.Logf("\n--- Testing Package %d: %s ---", j+1, pkgName) + var prList porchapi.PackageRevisionList + if err := t.client.List(t.ctx, &prList, client.InNamespace(t.testOptions.namespace)); err != nil { + t.T().Logf("failed to list package revisions for deletion: %v", err) + } else { + t.T().Logf("found %d package revisions to delete", len(prList.Items)) - pkgMetrics := createAndTestPackage(t, ctx, c, namespace, repoName, pkgName) - for _, m := range pkgMetrics { - recordMetric(m.Operation, repoName, pkgName, m.Duration.Seconds(), m.Error) + sort.Slice(prList.Items, func(i, j int) bool { + var ri, rj int + if ws := prList.Items[i].Spec.WorkspaceName; strings.Contains(ws, "v") { + _, _ = fmt.Sscanf(ws, "v%d", &ri) } - printIterationResults(t, logger, (i*(*numPackages))+j+1, pkgMetrics) - - allMetrics = append(allMetrics, TestMetrics{ - RepoName: repoName, - PkgName: pkgName, - Metrics: append(repoMetrics, pkgMetrics...), - }) - } - - // Cleanup repository - start := time.Now() - err := c.Delete(ctx, &configapi.Repository{ - ObjectMeta: metav1.ObjectMeta{ - Name: repoName, - Namespace: namespace, - }, + if ws := prList.Items[j].Spec.WorkspaceName; strings.Contains(ws, "v") { + _, _ = fmt.Sscanf(ws, "v%d", &rj) + } + return ri < rj }) - cleanupMetrics := []OperationMetrics{{ - Operation: "Delete Repository", - Duration: time.Since(start), - Error: err, - }} - printIterationResults(t, logger, (i+1)*(*numPackages), cleanupMetrics) - } - // Print consolidated results - printTestResults(t, logger, allMetrics) + *deletedCount = 0 + for _, pr := range prList.Items { + if err := t.ctx.Err(); err != nil { + t.T().Logf("Deletion interrupted: %v", err) + break + } + prefix := fmt.Sprintf("%s-test-", t.testOptions.namespace) + if !strings.HasPrefix(pr.Spec.RepositoryName, prefix) { + continue + } - // After all tests complete, print message and wait for interrupt - t.Log("\nTests completed. Prometheus server is running at http://localhost:9090") - t.Log("Press Ctrl+C to stop and cleanup...") + revisionNum := 1 + if strings.Contains(pr.Spec.WorkspaceName, "v") { + _, _ = fmt.Sscanf(pr.Spec.WorkspaceName, "v%d", &revisionNum) + } - // Wait for interrupt signal - <-sigChan - t.Log("\nReceived interrupt signal. Cleaning up...") -} + t.T().Logf("Deleting package revision: %s (repo: %s, pkg: %s, revision: %d)", + pr.Name, pr.Spec.RepositoryName, pr.Spec.PackageName, revisionNum) -func printIterationResults(t *testing.T, logger *TestLogger, iteration int, metrics []OperationMetrics) { - // Console output - t.Logf("\n=== Iteration %d Results ===", iteration) - t.Log("Operation Duration Status") - t.Log("--------------------------------------------------") - - // File output - logger.LogResult("\n=== Iteration %d Results ===", iteration) - logger.LogResult("Operation Duration Status") - logger.LogResult("--------------------------------------------------") - - for _, m := range metrics { - status := "Success" - if m.Error != nil { - status = "Failed: " + m.Error.Error() + if err = t.deletePackageRevision(pr.Spec.RepositoryName, pr.Spec.PackageName, pr.Name, revisionNum); err == nil { + proposeDel := t.metrics[pr.Spec.RepositoryName].pkgRevMetrics[pr.Spec.PackageName][revisionNum].Metrics[pkgRevProposeDeletion] + del := t.metrics[pr.Spec.RepositoryName].pkgRevMetrics[pr.Spec.PackageName][revisionNum].Metrics[pkgRevDelete] + t.resultsLogger.LogDeleted(pr.Name, proposeDel.Duration+del.Duration) + *deletedCount++ + } else { + t.T().Errorf("failed to delete package revision: %s (repo: %s, pkg: %s, revision: %d)", pr.Name, pr.Spec.RepositoryName, pr.Spec.PackageName, revisionNum) + } } - result := fmt.Sprintf("%-25s %-10v %s", - m.Operation, - m.Duration.Round(time.Millisecond), - status) + *deletionDuration = time.Since(*deletionStartTime) - t.Log(result) - logger.LogResult("%s", result) + t.T().Logf("Completed deletion of %d package revisions", deletedCount) } } -func printTestResults(t *testing.T, logger *TestLogger, allMetrics []TestMetrics) { +func (t *PerformanceTests) printTestResults(logger *TestLogger) { header := "\n=== Consolidated Performance Test Results ===" - t.Log(header) + t.T().Log(header) logger.LogResult("%s", header) - subheader := "Operation Min Max Avg Total" - t.Log(subheader) + subheader := "Operation Min Max Avg Total" + t.T().Log(subheader) logger.LogResult("%s", subheader) - divider := "------------------------------------------------------------------------" - t.Log(divider) + divider := "------------------------------------------------------------------------------------" + t.T().Log(divider) logger.LogResult("%s", divider) - stats := make(map[string]Stats) + operationStats := make(map[string]*Stats) + + operationHeadings := map[string]string{ + giteaRepoCreate: "Create Gitea Repository ", + porchRepoCreate: "Create Porch Repository ", + repoWait: "Repository Ready Wait", + pkgRevList: "Package Revision List", + pkgRevCreate: "Package Revision Create", + pkgRevResourcesGet: "Package Revision Get Resources", + pkgRevUpdate: "Package Revision Update", + pkgRevGet: "Package Revision Get", + pkgRevPropose: "Package Revision Propose", + pkgRevGetProposed: "Package Revision Get (Proposed)", + pkgRevPublished: "Package Revision Approve/Publish", + pkgRevProposeDeletion: "Package Revision Propose Deletion", + pkgRevDelete: "Package Revision Delete", + } + + repoOperations := []string{ + giteaRepoCreate, + porchRepoCreate, + repoWait, + } + + pkgRevOperations := []string{ + pkgRevList, + pkgRevCreate, + pkgRevResourcesGet, + pkgRevUpdate, + pkgRevGet, + pkgRevPropose, + pkgRevGetProposed, + pkgRevPublished, + pkgRevProposeDeletion, + pkgRevDelete, + } + + allOperations := append(repoOperations, pkgRevOperations...) + + for _, op := range allOperations { + operationStats[op] = &Stats{} + } - for _, m := range allMetrics { - for _, metric := range m.Metrics { - if metric.Error != nil { + t.metricsMutex.RLock() + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + repoMetrics, exists := t.metrics[repoName] + if !exists { + continue + } + + for key, repoOp := range repoMetrics.repoOps { + if repoOp.Error != nil { continue } - s := stats[metric.Operation] - if s.Count == 0 || metric.Duration < s.Min { - s.Min = metric.Duration + if stats, ok := operationStats[key]; ok { + if stats.Count == 0 || repoOp.Duration < stats.Min { + stats.Min = repoOp.Duration + } + if repoOp.Duration > stats.Max { + stats.Max = repoOp.Duration + } + stats.Total += repoOp.Duration + stats.Count++ } - if metric.Duration > s.Max { - s.Max = metric.Duration + } + + for j := 0; j < t.testOptions.numPkgs; j++ { + pkgName := fmt.Sprintf("network-function-%d", j) + pkgRevisions, exists := repoMetrics.pkgRevMetrics[pkgName] + if !exists { + continue + } + + for k := 1; k <= t.testOptions.numRevs; k++ { + pkgRevMetric, exists := pkgRevisions[k] + if !exists { + continue + } + + for opKey, opMetric := range pkgRevMetric.Metrics { + if opMetric.Error != nil { + continue + } + if stats, ok := operationStats[opKey]; ok { + if stats.Count == 0 || opMetric.Duration < stats.Min { + stats.Min = opMetric.Duration + } + if opMetric.Duration > stats.Max { + stats.Max = opMetric.Duration + } + stats.Total += opMetric.Duration + stats.Count++ + } + } } - s.Total += metric.Duration - s.Count++ - stats[metric.Operation] = s } } + t.metricsMutex.RUnlock() + + t.metricsMutex.RLock() + for i := 0; i < t.testOptions.numRepos; i++ { + for _, opKey := range repoOperations { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + repoMetrics, exists := t.metrics[repoName] + if !exists { + continue + } - for op, stat := range stats { - avg := stat.Total / time.Duration(stat.Count) - result := fmt.Sprintf("%-25s %-11v %-11v %-11v %-11v", - op, - stat.Min.Round(time.Millisecond), - stat.Max.Round(time.Millisecond), - avg.Round(time.Millisecond), - stat.Total.Round(time.Millisecond)) + repoOp, exists := repoMetrics.repoOps[opKey] + if !exists || repoOp.Error != nil { + continue + } - t.Log(result) - logger.LogResult("%s", result) + heading := operationHeadings[opKey] + if heading == "" { + heading = opKey + } + headingWithNum := fmt.Sprintf("%s R%d", heading, i) + result := fmt.Sprintf("%-37s %-11v %-11v %-11v %-11v", + headingWithNum, + repoOp.Duration.Round(time.Millisecond), + repoOp.Duration.Round(time.Millisecond), + repoOp.Duration.Round(time.Millisecond), + repoOp.Duration.Round(time.Millisecond)) + t.T().Log(result) + logger.LogResult("%s", result) + } + } + t.metricsMutex.RUnlock() + + for _, opKey := range pkgRevOperations { + stats := operationStats[opKey] + if stats.Count == 0 { + continue + } + + for k := 1; k <= t.testOptions.numRevs; k++ { + revStats := &Stats{} + revCount := 0 + + t.metricsMutex.RLock() + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + repoMetrics, exists := t.metrics[repoName] + if !exists { + continue + } + + for j := 0; j < t.testOptions.numPkgs; j++ { + pkgName := fmt.Sprintf("network-function-%d", j) + pkgRevisions, exists := repoMetrics.pkgRevMetrics[pkgName] + if !exists { + continue + } + + pkgRevMetric, exists := pkgRevisions[k] + if !exists { + continue + } + + opMetric, exists := pkgRevMetric.Metrics[opKey] + if !exists || opMetric.Error != nil { + continue + } + + if revCount == 0 || opMetric.Duration < revStats.Min { + revStats.Min = opMetric.Duration + } + if opMetric.Duration > revStats.Max { + revStats.Max = opMetric.Duration + } + revStats.Total += opMetric.Duration + revCount++ + } + } + t.metricsMutex.RUnlock() + + if revCount > 0 { + avg := revStats.Total / time.Duration(revCount) + heading := operationHeadings[opKey] + if heading == "" { + heading = opKey + } + headingWithRev := fmt.Sprintf("%s v%d", heading, k) + result := fmt.Sprintf("%-37s %-11v %-11v %-11v %-11v", + headingWithRev, + revStats.Min.Round(time.Millisecond), + revStats.Max.Round(time.Millisecond), + avg.Round(time.Millisecond), + revStats.Total.Round(time.Millisecond)) + t.T().Log(result) + logger.LogResult("%s", result) + } + } } - // Print errors if any hasErrors := false - for _, m := range allMetrics { - for _, metric := range m.Metrics { - if metric.Error != nil { + for i := 0; i < t.testOptions.numRepos; i++ { + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, i) + testMetric, exists := t.metrics[repoName] + if !exists { + continue + } + + for _, opMetric := range testMetric.repoOps { + if opMetric.Error != nil { if !hasErrors { errHeader := "\n=== Errors Encountered ===" - t.Log(errHeader) + t.T().Log(errHeader) logger.LogResult("%s", errHeader) hasErrors = true } - errMsg := fmt.Sprintf("Repository: %s, Package: %s, Operation: %s, Error: %v", - m.RepoName, m.PkgName, metric.Operation, metric.Error) - t.Log(errMsg) + errMsg := fmt.Sprintf("Repository: %s, Operation: %s, Error: %v", + repoName, opMetric.Operation, opMetric.Error) + t.T().Log(errMsg) logger.LogResult("%s", errMsg) } } + + for j := 0; j < t.testOptions.numPkgs; j++ { + pkgName := fmt.Sprintf("network-function-%d", j) + pkgRevisions, exists := testMetric.pkgRevMetrics[pkgName] + if !exists { + continue + } + + for k := 1; k <= t.testOptions.numRevs; k++ { + pkgRevMetric, exists := pkgRevisions[k] + if !exists { + continue + } + + for _, opMetric := range pkgRevMetric.Metrics { + if opMetric.Error != nil { + if !hasErrors { + errHeader := "\n=== Errors Encountered ===" + t.T().Log(errHeader) + logger.LogResult("%s", errHeader) + hasErrors = true + } + errMsg := fmt.Sprintf("Repository: %s, Package: %s, Revision: %d, Operation: %s, Error: %v", + repoName, pkgRevMetric.pkgName, k, opMetric.Operation, opMetric.Error) + t.T().Log(errMsg) + logger.LogResult("%s", errMsg) + } + } + } + } } } -func TestMain(m *testing.M) { - // Try to load kube config from standard locations - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - home, err := os.UserHomeDir() - if err == nil { - kubeconfig = filepath.Join(home, ".kube", "config") +func (t *PerformanceTests) logResults(lifecycleDuration time.Duration, deletedCount *int) { + if err := t.testLogger.Sync(); err != nil { + t.T().Logf("Warning: Failed to sync test logger: %v", err) + } + + t.T().Logf("Total lifecycle duration for all operations: %v", lifecycleDuration) + t.resultsLogger.LogToFile("Total lifecycle duration for all operations: %v", lifecycleDuration) + + t.T().Log("\nGenerating CSV results...") + if err := t.generateCSVResults(); err != nil { + t.T().Logf("Warning: Failed to generate CSV results: %v", err) + } else { + t.T().Logf("- CSV results saved to: %s", t.csvOptions.lifecycleCSV) + } + + if err := t.generateDetailedOperationsCSV(); err != nil { + t.T().Logf("Warning: Failed to generate detailed operations CSV: %v", err) + } else { + t.T().Logf("- Detailed operations CSV saved to: %s", t.csvOptions.operationsCSV) + } + + if t.testOptions.enableDeletion && *deletedCount > 0 { + if err := t.generateDeletionOperationsCSV(); err != nil { + t.T().Logf("Warning: Failed to generate deletion operations CSV: %v", err) + } else { + t.T().Logf("- Deletion operations CSV saved to: %s", t.csvOptions.deletionCSV) } } - if _, err := os.Stat(kubeconfig); err == nil { - os.Setenv("KUBERNETES_MASTER", "http://localhost:8080") + t.T().Logf("- Raw results saved to: %s", t.logOptions.resultsFile) + t.T().Logf("- Detailed log saved to: %s", t.logOptions.fullLogFile) + + if err := t.resultsLogger.Sync(); err != nil { + t.T().Logf("Warning: Failed to sync results logger: %v", err) + } + if err := t.testLogger.Sync(); err != nil { + t.T().Logf("Warning: Failed to sync test logger: %v", err) + } + + t.T().Log("\nTests completed!") +} + +func (t *PerformanceTests) processRepository(repoIndex, numRevs int, errorCalculator func(err error, errCount, numRevs int) bool) { + if err := t.ctx.Err(); err != nil { + return + } + + repoName := fmt.Sprintf("%s-test-%d", t.testOptions.namespace, repoIndex) + t.T().Logf("\n=== Creating Repository %d: %s ===", repoIndex+1, repoName) + t.createAndSetupRepo(repoName) + if err := t.ctx.Err(); err != nil { + return } - os.Exit(m.Run()) + t.metricsMutex.RLock() + for _, op := range t.metrics[repoName].repoOps { + t.resultsLogger.LogToFile("%s: %s - %v (took %.3fs)", repoName, op.Operation, op.Error, op.Duration.Seconds()) + } + t.metricsMutex.RUnlock() + + processPackage := func(pkgIndex int) { + if err := t.ctx.Err(); err != nil { + return + } + + errCount := 0 + pkgName := fmt.Sprintf("network-function-%d", pkgIndex) + t.T().Logf("\n--- Creating Package %s:%d ---", repoName, pkgIndex+1) + + t.metricsMutex.Lock() + t.metrics[repoName].pkgRevMetrics[pkgName] = make(map[int]PackageRevisionMetrics) + t.metricsMutex.Unlock() + + for k := 1; k <= numRevs; k++ { + if err := t.ctx.Err(); err != nil { + return + } + t.T().Logf("Creating revision %d/%d for package %s", k, t.testOptions.numRevs, pkgName) + if pkgRevName, err := t.doLifecycle(repoName, pkgName, k); err == nil { + t.metricsMutex.RLock() + for _, op := range t.metrics[repoName].pkgRevMetrics[pkgName][k].Metrics { + if op.Operation == fmt.Sprintf("%s:%d", pkgRevPublished, k) { + t.resultsLogger.LogApproved(repoName, pkgName, k, pkgRevName, op.Duration) + } else { + t.resultsLogger.LogToFile("%s:%s:%d - %s (took %.3fs)", repoName, pkgName, k, op.Operation, op.Duration.Seconds()) + } + } + t.metricsMutex.RUnlock() + } else { + t.T().Logf("An error occured during the creation/update of the package revision %s: %s", pkgRevName, err) + errCount++ + if errorCalculator(err, errCount, k) { + break + } + } + } + } + + pkgSemaphore := make(chan struct{}, t.testOptions.packageParallelism) + var pkgWg sync.WaitGroup + + for j := 0; j < t.testOptions.numPkgs; j++ { + if err := t.ctx.Err(); err != nil { + break + } + pkgSemaphore <- struct{}{} + pkgWg.Add(1) + go func(pkgIndex int) { + defer pkgWg.Done() + defer func() { <-pkgSemaphore }() + processPackage(pkgIndex) + }(j) + } + pkgWg.Wait() } diff --git a/test/performance/prometheus_metrics.go b/test/performance/prometheus_metrics.go deleted file mode 100644 index 1d829c31e..000000000 --- a/test/performance/prometheus_metrics.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2024, 2026 The kpt Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - // Operation duration metrics - operationDuration = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "porch_operation_duration_seconds", - Help: "Duration of Porch operations in seconds", - Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60}, - }, - []string{"operation", "repository", "package", "status"}, - ) - - // Operation counter metrics - operationCounter = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "porch_operations_total", - Help: "Total number of Porch operations", - }, - []string{"operation", "repository", "package", "status"}, - ) - - // Repository metrics - repositoryCounter = promauto.NewCounter( - prometheus.CounterOpts{ - Name: "porch_repositories_created_total", - Help: "Total number of repositories created", - }, - ) - - // Package metrics - packageCounter = promauto.NewCounter( - prometheus.CounterOpts{ - Name: "porch_packages_created_total", - Help: "Total number of packages created", - }, - ) -) - -// recordMetric records both duration and count for an operation -func recordMetric(operation, repoName, pkgName string, duration float64, err error) { - status := "success" - if err != nil { - status = "error" - } - - operationDuration.WithLabelValues(operation, repoName, pkgName, status).Observe(duration) - operationCounter.WithLabelValues(operation, repoName, pkgName, status).Inc() -} diff --git a/test/performance/promql_queries.txt b/test/performance/promql_queries.txt deleted file mode 100644 index 1029083b5..000000000 --- a/test/performance/promql_queries.txt +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2024 The kpt Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -Here are the PromQL queries to get operation timings for iterations and specific package operations: - -• Time Taken for Each Operation per Iteration -# Basic operation duration -porch_operation_duration_seconds_sum{operation="Create PackageRevision"} - -# Detailed breakdown by operation -sum by (operation) (porch_operation_duration_seconds_sum) - -# Operation duration with package and repository context -sum by (operation, package, repository) (porch_operation_duration_seconds_sum) - -# Latest operation durations -sort_desc(porch_operation_duration_seconds_sum) - - -• Specific Package Revision Operations -# Time taken for specific package revision creation -porch_operation_duration_seconds_sum{operation="Create PackageRevision", package="test-package-0"} - -# Time for package to move to proposed state -porch_operation_duration_seconds_sum{operation="Update to Proposed", package="test-package-0"} - -# Time for package to move to published state -porch_operation_duration_seconds_sum{operation="Update to Published", package="test-package-0"} - -# Time taken for package deletion -porch_operation_duration_seconds_sum{operation="Delete PackageRevision", package="test-package-0"} - - -• Comparative Analysis -# Compare durations across different operations for same package -sum by (operation) ( - porch_operation_duration_seconds_sum{package="test-package-0"} -) - -# Average duration per operation type -rate(porch_operation_duration_seconds_sum[5m]) / rate(porch_operation_duration_seconds_count[5m]) - -# Operation duration distribution -histogram_quantile(0.95, - sum by (le, operation) ( - rate(porch_operation_duration_seconds_bucket{package="test-package-0"}[5m]) - ) -) - - -• Time Series Analysis -# Operation duration over time -rate(porch_operation_duration_seconds_sum{operation="Create PackageRevision"}[5m]) - -# Compare operation times across iterations -sum by (operation) ( - increase(porch_operation_duration_seconds_sum[1h]) -) - - -• Success/Failure Analysis -# Duration of successful operations -porch_operation_duration_seconds_sum{status="success", operation="Create PackageRevision"} - -# Duration of failed operations -porch_operation_duration_seconds_sum{status="error", operation="Create PackageRevision"} - - -Example Usage: - - -• For a specific package operation: -# Get exact duration for creating package "test-package-0" -porch_operation_duration_seconds_sum{ - operation="Create PackageRevision", - package="test-package-0", - repository="porch-metrics-test-0" -} - -# Get full lifecycle timing for package "test-package-0" -sum by (operation) ( - porch_operation_duration_seconds_sum{ - package="test-package-0", - repository="porch-metrics-test-0" - } -) - - -2. For iteration analysis: -# Get timing for all operations in latest iteration -sum by (operation, package) ( - porch_operation_duration_seconds_sum{ - repository="porch-metrics-test-0" - } -) - -# Compare operation durations across iterations -rate(porch_operation_duration_seconds_sum[5m]) - / -rate(porch_operation_duration_seconds_count[5m]) - - -• For specific operation analysis: -# Detailed timing for package state transitions -sum by (operation) ( - porch_operation_duration_seconds_sum{ - operation=~"Update to.*", - package="test-package-0" - } -) - - - diff --git a/test/performance/resources/Kptfile b/test/performance/resources/Kptfile new file mode 100644 index 000000000..0464fddca --- /dev/null +++ b/test/performance/resources/Kptfile @@ -0,0 +1,13 @@ +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: CHANGE_ME + annotations: + config.kubernetes.io/local-config: "true" +info: + description: network function CHANGE_ME blueprint +pipeline: + mutators: + - image: REGISTRY_URL/set-namespace:v0.4.1 + configMap: + namespace: CHANGE_ME \ No newline at end of file diff --git a/test/performance/resources/deployment.yaml b/test/performance/resources/deployment.yaml new file mode 100644 index 000000000..66c3fe5fa --- /dev/null +++ b/test/performance/resources/deployment.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: CHANGE_ME +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: CHANGE_ME + template: + metadata: + labels: + app.kubernetes.io/name: CHANGE_ME + spec: + containers: + - name: nginx + image: nginx:latest \ No newline at end of file diff --git a/test/performance/types.go b/test/performance/types.go index 69bcdb932..0c07fa027 100644 --- a/test/performance/types.go +++ b/test/performance/types.go @@ -1,4 +1,4 @@ -// Copyright 2024, 2026 The kpt Authors +// Copyright 2024 The Nephio Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,27 +11,47 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package metrics import ( "time" ) -// OperationMetrics holds metrics for a single operation +const ( + giteaRepoCreate = "GITEA-REPO-CREATE" + porchRepoCreate = "PORCH-REPO-CREATE" + repoWait = "REPO-WAIT" + pkgRevList = "LIST" + pkgRevGet = "GET" + pkgRevGetProposed = "GET-PROPOSED" + pkgRevResourcesGet = "GET-RESOURCES" + pkgRevCreate = "CREATE" + pkgRevUpdate = "UPDATE" + pkgRevPropose = "PROPOSE" + pkgRevPublished = "APPROVE" + pkgRevProposeDeletion = "PROPOSE-DELETION" + pkgRevDelete = "DELETE" +) + type OperationMetrics struct { Operation string Duration time.Duration Error error + Timestamp time.Time // When the operation started } -// TestMetrics holds metrics for a test iteration type TestMetrics struct { - RepoName string - PkgName string - Metrics []OperationMetrics + RepoName string + repoOps map[string]OperationMetrics + pkgRevMetrics map[string]map[int]PackageRevisionMetrics } -// Stats holds statistics for operations +type PackageRevisionMetrics struct { + pkgName string + Revision int + Metrics map[string]OperationMetrics +} type Stats struct { Min time.Duration Max time.Duration From 81f791efdcae6d07a9f9ea98f55c60c882d7efbb Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Fri, 17 Jul 2026 23:21:38 +0200 Subject: [PATCH 02/13] Add new Function Pod resource usage metrics + expand on the package available for perf test Signed-off-by: Rendre Greyling --- .gitignore | 3 + .../grafana-resource-usage-dashboard.json | 2531 ++- .../metrics-resources/prometheus-config.yaml | 1 - deployments/porch/3-porch-server.yaml | 2 +- test/performance/csv_generation.go | 2 +- test/performance/logger.go | 2 +- .../packages/complex-package/Kptfile | 117 + .../complex-package/apply-replacements.yaml | 44 + .../packages/complex-package/configmap.yaml | 19 + .../packages/complex-package/deployment.yaml | 58 + .../gatekeeper-constraints.yaml | 53 + .../packages/complex-package/hpa.yaml | 32 + .../packages/complex-package/ingress.yaml | 31 + .../packages/complex-package/limitrange.yaml | 32 + .../complex-package/networkpolicy.yaml | 38 + .../complex-package/poddisruptionbudget.yaml | 17 + .../packages/complex-package/role.yaml | 22 + .../packages/complex-package/rolebinding.yaml | 20 + .../packages/complex-package/service.yaml | 21 + .../complex-package/serviceaccount.yaml | 12 + .../complex-package/starlark-run.yaml | 23 + .../packages/large-package/Kptfile | 129 + .../large-package/apply-replacements.yaml | 39 + .../packages/large-package/deployment.yaml | 17220 ++++++++++++++++ .../large-package/gatekeeper-constraints.yaml | 53 + .../packages/large-package/limitrange.yaml | 37 + .../packages/large-package/starlark-run.yaml | 38 + .../packages/small-package/Kptfile | 16 + .../packages/small-package/deployment.yaml | 20 + test/performance/performance_suite.go | 148 +- test/performance/porch_metrics_test.go | 2 +- test/performance/resources/Kptfile | 13 - test/performance/resources/deployment.yaml | 17 - test/performance/types.go | 2 +- 34 files changed, 20167 insertions(+), 647 deletions(-) create mode 100644 test/performance/packages/complex-package/Kptfile create mode 100644 test/performance/packages/complex-package/apply-replacements.yaml create mode 100644 test/performance/packages/complex-package/configmap.yaml create mode 100644 test/performance/packages/complex-package/deployment.yaml create mode 100644 test/performance/packages/complex-package/gatekeeper-constraints.yaml create mode 100644 test/performance/packages/complex-package/hpa.yaml create mode 100644 test/performance/packages/complex-package/ingress.yaml create mode 100644 test/performance/packages/complex-package/limitrange.yaml create mode 100644 test/performance/packages/complex-package/networkpolicy.yaml create mode 100644 test/performance/packages/complex-package/poddisruptionbudget.yaml create mode 100644 test/performance/packages/complex-package/role.yaml create mode 100644 test/performance/packages/complex-package/rolebinding.yaml create mode 100644 test/performance/packages/complex-package/service.yaml create mode 100644 test/performance/packages/complex-package/serviceaccount.yaml create mode 100644 test/performance/packages/complex-package/starlark-run.yaml create mode 100644 test/performance/packages/large-package/Kptfile create mode 100644 test/performance/packages/large-package/apply-replacements.yaml create mode 100644 test/performance/packages/large-package/deployment.yaml create mode 100644 test/performance/packages/large-package/gatekeeper-constraints.yaml create mode 100644 test/performance/packages/large-package/limitrange.yaml create mode 100644 test/performance/packages/large-package/starlark-run.yaml create mode 100644 test/performance/packages/small-package/Kptfile create mode 100644 test/performance/packages/small-package/deployment.yaml delete mode 100644 test/performance/resources/Kptfile delete mode 100644 test/performance/resources/deployment.yaml diff --git a/.gitignore b/.gitignore index 4facf47e5..b0d6d0bfa 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ __debug* ### Jetbrains IDEs ### .idea/* +### Cursor IDEs ### +.cursor + ### Hugo ### # Generated files by hugo docs/.hugo_build.lock diff --git a/deployments/metrics-resources/grafana-resource-usage-dashboard.json b/deployments/metrics-resources/grafana-resource-usage-dashboard.json index dc63a0d23..142b58313 100644 --- a/deployments/metrics-resources/grafana-resource-usage-dashboard.json +++ b/deployments/metrics-resources/grafana-resource-usage-dashboard.json @@ -1,5 +1,142 @@ { - "annotations": { "list": [] }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "expr": "(\n ((time() - max by (pod, container) (container_start_time_seconds{namespace=\"porch-system\", pod=~\"porch-server-.*\", container=\"porch-server\", image!=\"\"})) < 120) > 0\n) or (\n sum by (namespace, pod, container) (increase(container_oom_events_total{namespace=\"porch-system\", pod=~\"porch-server-.*\", container=\"porch-server\", image!=\"\"}[2m])) > 0\n)", + "filter": { + "exclude": false, + "ids": [ + 5, + 13 + ] + }, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "Porch Server - Pod Crashes", + "step": "10s", + "tagKeys": "namespace,pod,container", + "textFormat": "{{pod}}/{{container}} restarted or OOM-killed", + "titleFormat": "Pod crash", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "expr": "(\n ((time() - max by (pod, container) (container_start_time_seconds{namespace=\"porch-system\", pod=~\"function-runner-.*\", container=\"function-runner\", image!=\"\"})) < 120) > 0\n) or (\n sum by (namespace, pod, container) (increase(container_oom_events_total{namespace=\"porch-system\", pod=~\"function-runner-.*\", container=\"function-runner\", image!=\"\"}[2m])) > 0\n)", + "filter": { + "exclude": false, + "ids": [ + 6, + 15 + ] + }, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "Function Runner - Pod Crashes", + "step": "10s", + "tagKeys": "namespace,pod,container", + "textFormat": "{{pod}}/{{container}} restarted or OOM-killed", + "titleFormat": "Pod crash", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "expr": "(\n ((time() - max by (pod, container) (container_start_time_seconds{namespace=\"porch-system\", pod=~\"porch-controllers-.*\", container=\"porch-controllers\", image!=\"\"})) < 120) > 0\n) or (\n sum by (namespace, pod, container) (increase(container_oom_events_total{namespace=\"porch-system\", pod=~\"porch-controllers-.*\", container=\"porch-controllers\", image!=\"\"}[2m])) > 0\n)", + "filter": { + "exclude": false, + "ids": [ + 17, + 19 + ] + }, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "Porch Controllers - Pod Crashes", + "step": "10s", + "tagKeys": "namespace,pod,container", + "textFormat": "{{pod}}/{{container}} restarted or OOM-killed", + "titleFormat": "Pod crash", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "expr": "(\n ((time() - max by (pod, container) (container_start_time_seconds{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"})) < 120) > 0\n) or (\n sum by (namespace, pod, container) (increase(container_oom_events_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[2m])) > 0\n)", + "filter": { + "exclude": false, + "ids": [ + 32, + 34 + ] + }, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "PostgreSQL - Pod Crashes", + "step": "10s", + "tagKeys": "namespace,pod,container", + "textFormat": "{{pod}}/{{container}} restarted or OOM-killed", + "titleFormat": "Pod crash", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "expr": "(\n ((time() - max by (pod, container) (container_start_time_seconds{namespace=\"porch-fn-system\", container!=\"\", container!=\"POD\", image!=\"\"})) < 120) > 0\n) or (\n sum by (namespace, pod, container) (increase(container_oom_events_total{namespace=\"porch-fn-system\", container!=\"\", container!=\"POD\", image!=\"\"}[2m])) > 0\n)", + "filter": { + "exclude": false, + "ids": [ + 37, + 39, + 43 + ] + }, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "name": "Function Pods - Pod Crashes", + "step": "10s", + "tagKeys": "namespace,pod,container", + "textFormat": "{{pod}}/{{container}} restarted or OOM-killed", + "titleFormat": "Pod crash", + "type": "dashboard" + } + ] + }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -9,18 +146,575 @@ "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Resource Stats Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-server\"}[5m])", + "refId": "A" + } + ], + "title": "Porch Server CPU (cores)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 536870912 + }, + { + "color": "red", + "value": 1073741824 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{job=\"porch-server\"}", + "refId": "A" + } + ], + "title": "Porch Server Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(process_cpu_seconds_total{job=\"function-runner\"}[5m])", + "refId": "A" + } + ], + "title": "Function Runner CPU (cores)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 536870912 + }, + { + "color": "red", + "value": 1073741824 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{job=\"function-runner\"}", + "refId": "A" + } + ], + "title": "Function Runner Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", + "refId": "A" + } + ], + "title": "Porch Controllers CPU (cores)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 536870912 + }, + { + "color": "red", + "value": 1073741824 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", + "refId": "A" + } + ], + "title": "Porch Controllers Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current CPU usage for porch-postgresql (limit: 500m per deployment)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "short", + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[5m]))", + "refId": "A" + } + ], + "title": "PostgreSQL CPU (cores)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current memory working set for porch-postgresql (limit: 512Mi per deployment)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 268435456 + }, + { + "color": "red", + "value": 536870912 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"})", + "refId": "A" + } + ], + "title": "PostgreSQL Memory", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 1, "panels": [], "title": "CPU Usage", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "CPU usage rate for porch-server", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "CPU cores", @@ -31,26 +725,48 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, "id": 2, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "rate(process_cpu_seconds_total{job=\"porch-server\"}[5m])", "legendFormat": "{{instance}}", "refId": "A" @@ -60,11 +776,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "CPU usage rate for function-runner", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "CPU cores", @@ -75,26 +796,48 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, "id": 3, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "rate(process_cpu_seconds_total{job=\"function-runner\"}[5m])", "legendFormat": "{{instance}}", "refId": "A" @@ -104,11 +847,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "CPU usage for porch-controllers (shared process for repositories, packagerevisions, packagevariants, packagevariantsets, and functionconfigs reconcilers)", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "CPU cores", @@ -119,26 +867,48 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, "id": 16, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", "legendFormat": "{{instance}}", "refId": "A" @@ -147,20 +917,101 @@ "title": "Porch Controllers - CPU Usage", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "CPU usage for the porch-postgresql container (from cAdvisor via kubelet). postgres-exporter provides database query metrics only, not container CPU.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "CPU cores", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 31, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[5m])) by (pod)", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "PostgreSQL - CPU Usage", + "type": "timeseries" + }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, "id": 4, "panels": [], "title": "Memory Usage", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "Memory usage for porch-server", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "Memory", @@ -171,32 +1022,57 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, "id": 5, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "process_resident_memory_bytes{job=\"porch-server\"}", "legendFormat": "resident - {{instance}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "process_virtual_memory_bytes{job=\"porch-server\"}", "legendFormat": "virtual - {{instance}}", "refId": "B" @@ -206,11 +1082,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "Memory usage for function-runner", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "Memory", @@ -221,32 +1102,57 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, "id": 6, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "process_resident_memory_bytes{job=\"function-runner\"}", "legendFormat": "resident - {{instance}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "process_virtual_memory_bytes{job=\"function-runner\"}", "legendFormat": "virtual - {{instance}}", "refId": "B" @@ -256,11 +1162,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "Memory usage for porch-controllers (shared process for all reconcilers)", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "Memory", @@ -271,57 +1182,314 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, "id": 17, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", + "legendFormat": "resident - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_virtual_memory_bytes{job=\"porch-controllers\"}", + "legendFormat": "virtual - {{instance}}", + "refId": "B" + } + ], + "title": "Porch Controllers - Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Memory working set for the porch-postgresql container (from cAdvisor via kubelet).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Memory", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 32, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_working_set_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}", + "legendFormat": "working set - {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_usage_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}", + "legendFormat": "usage - {{pod}}", + "refId": "B" + } + ], + "title": "PostgreSQL - Memory Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 20, + "panels": [], + "title": "Porch Controllers - Per-Controller Workload", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Reconciliation backlog per controller (repositories=repository, packagerevisions=packagerevision, packagevariants, packagevariantsets, functionconfigs=functionconfig)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Items", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 44 + }, + "id": 21, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"porch-controllers\", name=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Workqueue Depth (all controllers)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Reconciliations per second per controller", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Reconciles/s", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 44 + }, + "id": 22, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", - "legendFormat": "resident - {{instance}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"porch-controllers\", controller=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}[5m])) by (controller)", + "legendFormat": "{{controller}}", "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "process_virtual_memory_bytes{job=\"porch-controllers\"}", - "legendFormat": "virtual - {{instance}}", - "refId": "B" } ], - "title": "Porch Controllers - Memory Usage", + "title": "Reconcile Rate (all controllers)", "type": "timeseries" }, { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, - "id": 7, - "panels": [], - "title": "Go Runtime Metrics", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Number of active goroutines", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "p99 reconciliation duration per controller", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "Goroutines", + "axisLabel": "Duration", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, @@ -329,55 +1497,70 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "unit": "short", + "unit": "s", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 19 }, - "id": 8, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 44 + }, + "id": 23, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_goroutines{job=\"porch-server\"}", - "legendFormat": "porch-server ({{instance}})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=\"porch-controllers\", controller=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}[5m])) by (le, controller))", + "legendFormat": "{{controller}} p99", "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_goroutines{job=\"function-runner\"}", - "legendFormat": "function-runner ({{instance}})", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_goroutines{job=\"porch-controllers\"}", - "legendFormat": "porch-controllers ({{instance}})", - "refId": "C" } ], - "title": "Goroutines", + "title": "Reconcile Duration p99 (all controllers)", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Go heap memory allocation", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Filtered view for the selected controller ($controller)", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "Memory", + "axisLabel": "Items", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, @@ -385,73 +1568,79 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "unit": "bytes", + "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 19 }, - "id": 9, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 24, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_memstats_heap_alloc_bytes{job=\"porch-server\"}", - "legendFormat": "porch-server heap alloc ({{instance}})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"porch-controllers\", name=\"$controller\"}", + "legendFormat": "depth", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_memstats_heap_inuse_bytes{job=\"porch-server\"}", - "legendFormat": "porch-server heap inuse ({{instance}})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"porch-controllers\", controller=\"$controller\"}[5m])) by (result)", + "legendFormat": "reconcile {{result}}", "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_memstats_heap_alloc_bytes{job=\"function-runner\"}", - "legendFormat": "function-runner heap alloc ({{instance}})", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_memstats_heap_inuse_bytes{job=\"function-runner\"}", - "legendFormat": "function-runner heap inuse ({{instance}})", - "refId": "D" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_memstats_heap_alloc_bytes{job=\"porch-controllers\"}", - "legendFormat": "porch-controllers heap alloc ({{instance}})", - "refId": "E" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_memstats_heap_inuse_bytes{job=\"porch-controllers\"}", - "legendFormat": "porch-controllers heap inuse ({{instance}})", - "refId": "F" } ], - "title": "Go Heap Memory", + "title": "$controller - Workload", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Garbage collection duration", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Shared porch-controllers process resource usage (same for all reconcilers)", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "Duration", + "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, @@ -459,143 +1648,164 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "unit": "s", + "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 19 }, - "id": 10, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 25, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(go_gc_duration_seconds_sum{job=\"porch-server\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"porch-server\"}[5m])", - "legendFormat": "porch-server avg GC ({{instance}})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", + "legendFormat": "CPU cores", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(go_gc_duration_seconds_sum{job=\"function-runner\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"function-runner\"}[5m])", - "legendFormat": "function-runner avg GC ({{instance}})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_goroutines{job=\"porch-controllers\"}", + "legendFormat": "goroutines", "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(go_gc_duration_seconds_sum{job=\"porch-controllers\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"porch-controllers\"}[5m])", - "legendFormat": "porch-controllers avg GC ({{instance}})", - "refId": "C" } ], - "title": "GC Duration (avg)", + "title": "Porch Controllers - Shared Process Resources", "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, - "id": 11, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 40, "panels": [], - "title": "Resource Stats Summary", + "title": "Function Pods - Resource Stats Summary", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 1 } - ] - }, - "unit": "short", - "decimals": 3 - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 28 }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, - "textMode": "auto" + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(process_cpu_seconds_total{job=\"porch-server\"}[5m])", - "refId": "A" - } - ], - "title": "Porch Server CPU (cores)", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Total number of running function pod containers in porch-fn-system", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 536870912 }, - { "color": "red", "value": 1073741824 } + { + "color": "blue", + "value": null + } ] }, - "unit": "bytes" + "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 28 }, - "id": 13, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 61 + }, + "id": 41, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, "textMode": "auto" }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "process_resident_memory_bytes{job=\"porch-server\"}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "count(container_memory_working_set_bytes{namespace=\"porch-fn-system\", container!=\"\", container!=\"POD\"})", "refId": "A" } ], - "title": "Porch Server Memory", + "title": "Function Pod Containers Running", "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total CPU used across all function pods", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 4 + } ] }, "unit": "short", @@ -603,76 +1813,134 @@ }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 28 }, - "id": 14, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 61 + }, + "id": 42, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, "textMode": "auto" }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(process_cpu_seconds_total{job=\"function-runner\"}[5m])", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-fn-system\", container!=\"\", container!=\"POD\"}[5m]))", "refId": "A" } ], - "title": "Function Runner CPU (cores)", + "title": "Function Pods Total CPU (cores)", "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total memory working set across all function pods", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 536870912 }, - { "color": "red", "value": 1073741824 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1073741824 + }, + { + "color": "red", + "value": 4294967296 + } ] }, "unit": "bytes" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 28 }, - "id": 15, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 61 + }, + "id": 43, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, "textMode": "auto" }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "process_resident_memory_bytes{job=\"function-runner\"}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"porch-fn-system\", container!=\"\", container!=\"POD\"})", "refId": "A" } ], - "title": "Function Runner Memory", + "title": "Function Pods Total Memory", "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Peak CPU usage by a single function pod container in the last 5 minutes", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "short", @@ -680,125 +1948,67 @@ }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 28 }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", - "refId": "A" - } - ], - "title": "Porch Controllers CPU (cores)", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 536870912 }, - { "color": "red", "value": 1073741824 } - ] - }, - "unit": "bytes" - }, - "overrides": [] + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 61 }, - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 28 }, - "id": 19, + "id": 44, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, - "textMode": "auto" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", - "refId": "A" - } - ], - "title": "Porch Controllers Memory", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, - "id": 20, - "panels": [], - "title": "Porch Controllers - Per-Controller Workload", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Reconciliation backlog per controller (repositories=repository, packagerevisions=packagerevision, packagevariants, packagevariantsets, functionconfigs=functionconfig)", - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisCenteredZero": false, - "axisLabel": "Items", - "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "unit": "short", - "min": 0 + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 33 }, - "id": 21, - "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "textMode": "auto" }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "workqueue_depth{job=\"porch-controllers\", name=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}", - "legendFormat": "{{name}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(rate(container_cpu_usage_seconds_total{namespace=\"porch-fn-system\", container!=\"\", container!=\"POD\"}[5m]))", "refId": "A" } ], - "title": "Workqueue Depth (all controllers)", - "type": "timeseries" + "title": "Function Pods Peak CPU (single container)", + "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Reconciliations per second per controller", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 35, + "panels": [], + "title": "Function Pods (porch-fn-system) \u2014 Container Resource Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "CPU usage rate per function pod container (cAdvisor). Covers all binaries regardless of language/runtime.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "Reconciles/s", + "axisLabel": "CPU cores", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, @@ -806,43 +2016,70 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 33 }, - "id": 22, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 36, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(controller_runtime_reconcile_total{job=\"porch-controllers\", controller=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}[5m])) by (controller)", - "legendFormat": "{{controller}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (\n rate(container_cpu_usage_seconds_total{\n namespace=\"porch-fn-system\",\n container!=\"\",\n container!=\"POD\"\n }[5m])\n)", + "legendFormat": "{{pod}} / {{container}}", "refId": "A" } ], - "title": "Reconcile Rate (all controllers)", + "title": "Function Pods \u2014 CPU Usage per Container", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "p99 reconciliation duration per controller", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Memory working set per function pod container (cAdvisor). Covers all binaries regardless of language/runtime.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "Duration", + "axisLabel": "Memory", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, @@ -850,151 +2087,234 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "unit": "s", + "unit": "bytes", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 33 }, - "id": 23, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 37, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=\"porch-controllers\", controller=~\"repository|packagerevision|packagevariant|packagevariantset|functionconfig\"}[5m])) by (le, controller))", - "legendFormat": "{{controller}} p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (\n container_memory_working_set_bytes{\n namespace=\"porch-fn-system\",\n container!=\"\",\n container!=\"POD\"\n }\n)", + "legendFormat": "{{pod}} / {{container}}", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (\n container_memory_rss{\n namespace=\"porch-fn-system\",\n container!=\"\",\n container!=\"POD\"\n }\n)", + "legendFormat": "RSS: {{pod}} / {{container}}", + "refId": "B" } ], - "title": "Reconcile Duration p99 (all controllers)", + "title": "Function Pods \u2014 Memory Usage per Container", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Filtered view for the selected controller ($controller)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "CPU usage rate per function pod, all containers summed (cAdvisor).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "Items", + "axisLabel": "CPU cores", "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, + "drawStyle": "bars", + "fillOpacity": 80, "gradientMode": "none", "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, - "id": 24, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 38, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "workqueue_depth{job=\"porch-controllers\", name=\"$controller\"}", - "legendFormat": "depth", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (\n rate(container_cpu_usage_seconds_total{\n namespace=\"porch-fn-system\",\n container!=\"\",\n container!=\"POD\"\n }[5m])\n)", + "legendFormat": "{{pod}}", "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(controller_runtime_reconcile_total{job=\"porch-controllers\", controller=\"$controller\"}[5m])) by (result)", - "legendFormat": "reconcile {{result}}", - "refId": "B" } ], - "title": "$controller - Workload", + "title": "Function Pods \u2014 CPU Usage per Pod (stacked)", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Shared porch-controllers process resource usage (same for all reconcilers)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Memory working set per function pod, all containers summed (cAdvisor).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "", + "axisLabel": "Memory", "axisPlacement": "auto", - "drawStyle": "line", - "fillOpacity": 10, + "drawStyle": "bars", + "fillOpacity": 80, "gradientMode": "none", "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "unit": "short", + "unit": "bytes", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, - "id": 25, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 39, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(process_cpu_seconds_total{job=\"porch-controllers\"}[5m])", - "legendFormat": "CPU cores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (\n container_memory_working_set_bytes{\n namespace=\"porch-fn-system\",\n container!=\"\",\n container!=\"POD\"\n }\n)", + "legendFormat": "{{pod}}", "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "go_goroutines{job=\"porch-controllers\"}", - "legendFormat": "goroutines", - "refId": "B" } ], - "title": "Porch Controllers - Shared Process Resources", + "title": "Function Pods \u2014 Memory Usage per Pod (stacked)", "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 }, - "id": 30, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 84 + }, + "id": 7, "panels": [], - "title": "PostgreSQL Resource Usage", + "title": "Go Runtime Metrics", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "CPU usage for the porch-postgresql container (from cAdvisor via kubelet). postgres-exporter provides database query metrics only, not container CPU.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of active goroutines", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, - "axisLabel": "CPU cores", + "axisLabel": "Goroutines", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, @@ -1002,40 +2322,85 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 }, - "id": 31, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 85 + }, + "id": 8, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[5m])) by (pod)", - "legendFormat": "{{pod}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_goroutines{job=\"porch-server\"}", + "legendFormat": "porch-server ({{instance}})", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_goroutines{job=\"function-runner\"}", + "legendFormat": "function-runner ({{instance}})", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_goroutines{job=\"porch-controllers\"}", + "legendFormat": "porch-controllers ({{instance}})", + "refId": "C" } ], - "title": "PostgreSQL - CPU Usage", + "title": "Goroutines", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Memory working set for the porch-postgresql container (from cAdvisor via kubelet).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Go heap memory allocation", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisCenteredZero": false, "axisLabel": "Memory", @@ -1046,139 +2411,238 @@ "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "bytes", "min": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, - "id": 32, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 85 + }, + "id": 9, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "container_memory_working_set_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}", - "legendFormat": "working set - {{pod}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"porch-server\"}", + "legendFormat": "porch-server heap alloc ({{instance}})", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "container_memory_usage_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}", - "legendFormat": "usage - {{pod}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_memstats_heap_inuse_bytes{job=\"porch-server\"}", + "legendFormat": "porch-server heap inuse ({{instance}})", "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"function-runner\"}", + "legendFormat": "function-runner heap alloc ({{instance}})", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_memstats_heap_inuse_bytes{job=\"function-runner\"}", + "legendFormat": "function-runner heap inuse ({{instance}})", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_memstats_heap_alloc_bytes{job=\"porch-controllers\"}", + "legendFormat": "porch-controllers heap alloc ({{instance}})", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "go_memstats_heap_inuse_bytes{job=\"porch-controllers\"}", + "legendFormat": "porch-controllers heap inuse ({{instance}})", + "refId": "F" } ], - "title": "PostgreSQL - Memory Usage", + "title": "Go Heap Memory", "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Current CPU usage for porch-postgresql (limit: 500m per deployment)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Garbage collection duration", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.25 }, - { "color": "red", "value": 0.5 } - ] + "color": { + "mode": "palette-classic" }, - "unit": "short", - "decimals": 3 + "custom": { + "axisCenteredZero": false, + "axisLabel": "Duration", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "s", + "min": 0 }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 58 }, - "id": 33, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 85 + }, + "id": 10, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, - "textMode": "auto" + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(go_gc_duration_seconds_sum{job=\"porch-server\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"porch-server\"}[5m])", + "legendFormat": "porch-server avg GC ({{instance}})", "refId": "A" - } - ], - "title": "PostgreSQL CPU (cores)", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Current memory working set for porch-postgresql (limit: 512Mi per deployment)", - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 268435456 }, - { "color": "red", "value": 536870912 } - ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "unit": "bytes" + "expr": "rate(go_gc_duration_seconds_sum{job=\"function-runner\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"function-runner\"}[5m])", + "legendFormat": "function-runner avg GC ({{instance}})", + "refId": "B" }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 58 }, - "id": 34, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"porch-system\", pod=~\"porch-postgresql-.*\", container=\"postgresql\", image!=\"\"})", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(go_gc_duration_seconds_sum{job=\"porch-controllers\"}[5m]) / rate(go_gc_duration_seconds_count{job=\"porch-controllers\"}[5m])", + "legendFormat": "porch-controllers avg GC ({{instance}})", + "refId": "C" } ], - "title": "PostgreSQL Memory", - "type": "stat" + "title": "GC Duration (avg)", + "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 38, "style": "dark", - "tags": ["porch", "resources", "kubernetes"], + "tags": [ + "porch", + "resources", + "kubernetes" + ], "templating": { "list": [ { - "current": { "selected": true, "text": "repository", "value": "repository" }, + "current": { + "selected": true, + "text": "repository", + "value": "repository" + }, "hide": 0, "includeAll": false, "label": "Controller", "multi": false, "name": "controller", "options": [ - { "selected": true, "text": "repository", "value": "repository" }, - { "selected": false, "text": "packagerevision", "value": "packagerevision" }, - { "selected": false, "text": "packagevariant", "value": "packagevariant" }, - { "selected": false, "text": "packagevariantset", "value": "packagevariantset" }, - { "selected": false, "text": "functionconfig", "value": "functionconfig" } + { + "selected": true, + "text": "repository", + "value": "repository" + }, + { + "selected": false, + "text": "packagerevision", + "value": "packagerevision" + }, + { + "selected": false, + "text": "packagevariant", + "value": "packagevariant" + }, + { + "selected": false, + "text": "packagevariantset", + "value": "packagevariantset" + }, + { + "selected": false, + "text": "functionconfig", + "value": "functionconfig" + } ], "query": "repository,packagerevision,packagevariant,packagevariantset,functionconfig", "skipUrlSync": false, @@ -1186,7 +2650,10 @@ } ] }, - "time": { "from": "now-15m", "to": "now" }, + "time": { + "from": "now-15m", + "to": "now" + }, "timepicker": {}, "timezone": "", "title": "Porch Resource Usage", diff --git a/deployments/metrics-resources/prometheus-config.yaml b/deployments/metrics-resources/prometheus-config.yaml index 5004e5283..97ebfd7ec 100644 --- a/deployments/metrics-resources/prometheus-config.yaml +++ b/deployments/metrics-resources/prometheus-config.yaml @@ -56,7 +56,6 @@ scrape_configs: service: 'porch' component: 'postgresql' - # Container CPU/memory for porch-postgresql (and other porch-system pods). - job_name: 'kubernetes-cadvisor' honor_labels: true scrape_interval: 10s diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 4108579e8..94fc7307a 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -66,7 +66,7 @@ spec: memory: 256Mi cpu: 250m limits: - memory: 512Mi + memory: 2Gi volumeMounts: - mountPath: /cache name: cache-volume diff --git a/test/performance/csv_generation.go b/test/performance/csv_generation.go index b0d549377..2aebee43d 100644 --- a/test/performance/csv_generation.go +++ b/test/performance/csv_generation.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The Nephio Authors +// Copyright 2026 The Kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/performance/logger.go b/test/performance/logger.go index 8b10b7323..e84ca13db 100644 --- a/test/performance/logger.go +++ b/test/performance/logger.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The Nephio Authors +// Copyright 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/performance/packages/complex-package/Kptfile b/test/performance/packages/complex-package/Kptfile new file mode 100644 index 000000000..84d4f41ef --- /dev/null +++ b/test/performance/packages/complex-package/Kptfile @@ -0,0 +1,117 @@ +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: perf-test-complex-package + annotations: + config.kubernetes.io/local-config: "true" +info: + description: Complex test package for Porch performance testing - uses every kpt catalog function across 10 resource files +pipeline: + mutators: + # 1. set-namespace — stamps the target namespace on every namespaced resource + - image: CHANGE_IMAGE/set-namespace:v0.4.1 + configMap: + namespace: CHANGE_NAMESPACE + # 2. create-setters — scans resources and inserts # kpt-set: ${name} setter comments + - image: CHANGE_IMAGE/create-setters:v0.1.0 + configMap: + app-name: complex-app + environment: dev + image: docker.io/nginx:latest + replicas: "2" + port: "8080" + target-port: "80" + ingress-host: complex-app.example.com + cpu-request: 100m + cpu-limit: 500m + memory-request: 128Mi + memory-limit: 256Mi + min-replicas: "2" + max-replicas: "10" + team: platform + version: v1.0.0 + # 3. apply-setters — substitutes all # kpt-set: ${name} markers with configured values + - image: CHANGE_IMAGE/apply-setters:v0.2.0 + configMap: + app-name: complex-app + environment: dev + image: docker.io/nginx:latest + replicas: "2" + port: "8080" + target-port: "80" + ingress-host: complex-app.example.com + cpu-request: 100m + cpu-limit: 500m + memory-request: 128Mi + memory-limit: 256Mi + min-replicas: "2" + max-replicas: "10" + team: platform + version: v1.0.0 + # 4. set-labels — bulk-stamps labels onto every resource + - image: CHANGE_IMAGE/set-labels:v0.2.0 + configMap: + app: complex-app + environment: dev + managed-by: kpt + team: platform + version: v1.0.0 + # 5. set-annotations — bulk-stamps annotations onto every resource + - image: CHANGE_IMAGE/set-annotations:v0.1.4 + configMap: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: perf-test-suite + team: platform + # 6. set-image — updates the nginx container image name and tag + - image: CHANGE_IMAGE/set-image:v0.1.1 + configMap: + name: nginx + newName: docker.io/nginx + newTag: latest + # 7. search-replace — finds a specific field path and overwrites its value + - image: CHANGE_IMAGE/search-replace:v0.2.0 + configMap: + by-path: metadata.annotations.[app.kubernetes.io/managed-by] + put-value: kpt + # 8. ensure-name-substring — prepends 'complex-app-' to any resource whose + # name does not already start with that prefix + - image: CHANGE_IMAGE/ensure-name-substring:v0.1.1 + configMap: + prepend: complex-app- + selectors: + - kind: Deployment + - kind: Service + - kind: Ingress + - kind: ConfigMap + - kind: ServiceAccount + - kind: Role + - kind: RoleBinding + - kind: NetworkPolicy + - kind: HorizontalPodAutoscaler + - kind: PodDisruptionBudget + - kind: LimitRange + # 9. apply-replacements — propagates field values across resources as defined + # in the ApplyReplacements resource inside the package + - image: CHANGE_IMAGE/apply-replacements:v0.1.1 + configPath: apply-replacements.yaml + # 10. starlark — executes the StarlarkRun script from the package + - image: CHANGE_IMAGE/starlark:v0.4.3 + configPath: starlark-run.yaml + # 11. upsert-resource — inserts or updates the LimitRange into the package + - image: CHANGE_IMAGE/upsert-resource:v0.2.0 + configPath: limitrange.yaml + # 12. generate-folders — generates Config Controller Folder resources; no-op + # when no HierarchyConfig resources are present in the package + - image: CHANGE_IMAGE/generate-folders:v0.1.1 + # 13. set-enforcement-action — sets enforcementAction: warn on all Constraint + # resources (defined in gatekeeper-constraints.yaml) + - image: CHANGE_IMAGE/set-enforcement-action:v0.1.1 + configMap: + enforcementAction: warn + validators: + # 14. kubeconform — validates KRM resources against their JSON schemas + # (uses baked-in schemas; see https://catalog.kpt.dev/kubeconform/v0.1/) + - image: CHANGE_IMAGE/kubeconform:v0.1.1 + # 15. gatekeeper — validates resources against ConstraintTemplate/Constraint + # policies defined in gatekeeper-constraints.yaml + - image: CHANGE_IMAGE/gatekeeper:v0.2.1 diff --git a/test/performance/packages/complex-package/apply-replacements.yaml b/test/performance/packages/complex-package/apply-replacements.yaml new file mode 100644 index 000000000..d6fbdd6aa --- /dev/null +++ b/test/performance/packages/complex-package/apply-replacements.yaml @@ -0,0 +1,44 @@ +apiVersion: fn.kpt.dev/v1alpha1 +kind: ApplyReplacements +metadata: + name: apply-replacements + annotations: + config.kubernetes.io/local-config: "true" +replacements: + # Propagate APP_ENV from the ConfigMap into every Deployment's template label + - source: + kind: ConfigMap + name: complex-app-config + fieldPath: data.APP_ENV + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.metadata.labels.environment + options: + create: true + # Propagate APP_VERSION from the ConfigMap into every Deployment's template label + - source: + kind: ConfigMap + name: complex-app-config + fieldPath: data.APP_VERSION + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.metadata.labels.version + options: + create: true + # Propagate APP_NAME from the ConfigMap into the Ingress host annotation + - source: + kind: ConfigMap + name: complex-app-config + fieldPath: data.APP_NAME + targets: + - select: + kind: Ingress + fieldPaths: + - metadata.annotations.[app.kubernetes.io/name] + options: + create: true + diff --git a/test/performance/packages/complex-package/configmap.yaml b/test/performance/packages/complex-package/configmap.yaml new file mode 100644 index 000000000..37a3dbee5 --- /dev/null +++ b/test/performance/packages/complex-package/configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: complex-app-config # kpt-set: ${app-name}-config + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +data: + APP_ENV: dev # kpt-set: ${environment} + APP_PORT: "8080" # kpt-set: ${port} + APP_NAME: complex-app # kpt-set: ${app-name} + APP_VERSION: v1.0.0 # kpt-set: ${version} + LOG_LEVEL: info + METRICS_ENABLED: "true" + diff --git a/test/performance/packages/complex-package/deployment.yaml b/test/performance/packages/complex-package/deployment.yaml new file mode 100644 index 000000000..60f22133e --- /dev/null +++ b/test/performance/packages/complex-package/deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +spec: + replicas: 2 # kpt-set: ${replicas} + selector: + matchLabels: + app: complex-app # kpt-set: ${app-name} + template: + metadata: + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + spec: + serviceAccountName: complex-app # kpt-set: ${app-name} + containers: + - name: nginx + image: docker.io/nginx:latest # kpt-set: ${image} + ports: + - containerPort: 80 # kpt-set: ${target-port} + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + env: + - name: APP_ENV + valueFrom: + configMapKeyRef: + name: complex-app-config # kpt-set: ${app-name}-config + key: APP_ENV + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + livenessProbe: + httpGet: + path: /healthz + port: 80 # kpt-set: ${target-port} + initialDelaySeconds: 10 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /ready + port: 80 # kpt-set: ${target-port} + initialDelaySeconds: 5 + periodSeconds: 10 + diff --git a/test/performance/packages/complex-package/gatekeeper-constraints.yaml b/test/performance/packages/complex-package/gatekeeper-constraints.yaml new file mode 100644 index 000000000..2fcda58a7 --- /dev/null +++ b/test/performance/packages/complex-package/gatekeeper-constraints.yaml @@ -0,0 +1,53 @@ +--- +# ConstraintTemplate: defines the OPA Rego policy that checks for required labels. +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: requiredlabels + annotations: + config.kubernetes.io/local-config: "true" +spec: + crd: + spec: + names: + kind: RequiredLabels + validation: + openAPIV3Schema: + type: object + properties: + labels: + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package requiredlabels + + violation[{"msg": msg, "details": {"missing_labels": missing}}] { + provided := {label | input.review.object.metadata.labels[label]} + required := {label | label := input.parameters.labels[_]} + missing := required - provided + count(missing) > 0 + msg := sprintf("you must provide labels: %v", [missing]) + } +--- +# Constraint: enforces that all Deployments carry the required labels. +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: RequiredLabels +metadata: + name: must-have-required-labels + annotations: + config.kubernetes.io/local-config: "true" +spec: + enforcementAction: warn + match: + kinds: + - apiGroups: ["apps"] + kinds: ["Deployment"] + parameters: + labels: + - app + - environment + - team + diff --git a/test/performance/packages/complex-package/hpa.yaml b/test/performance/packages/complex-package/hpa.yaml new file mode 100644 index 000000000..39fffc123 --- /dev/null +++ b/test/performance/packages/complex-package/hpa.yaml @@ -0,0 +1,32 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: complex-app # kpt-set: ${app-name} + minReplicas: 2 # kpt-set: ${min-replicas} + maxReplicas: 10 # kpt-set: ${max-replicas} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + diff --git a/test/performance/packages/complex-package/ingress.yaml b/test/performance/packages/complex-package/ingress.yaml new file mode 100644 index 000000000..a3cd6df95 --- /dev/null +++ b/test/performance/packages/complex-package/ingress.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "true" +spec: + ingressClassName: nginx + rules: + - host: complex-app.example.com # kpt-set: ${ingress-host} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: complex-app # kpt-set: ${app-name} + port: + number: 8080 # kpt-set: ${port} + tls: + - hosts: + - complex-app.example.com # kpt-set: ${ingress-host} + secretName: complex-app-tls # kpt-set: ${app-name}-tls + diff --git a/test/performance/packages/complex-package/limitrange.yaml b/test/performance/packages/complex-package/limitrange.yaml new file mode 100644 index 000000000..8ebbfce7c --- /dev/null +++ b/test/performance/packages/complex-package/limitrange.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: LimitRange +metadata: + name: complex-app-limits + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + team: platform # kpt-set: ${team} + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: perf-test-suite +spec: + limits: + - type: Container + default: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + defaultRequest: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + max: + cpu: "2" + memory: 1Gi + min: + cpu: 10m + memory: 16Mi + - type: Pod + max: + cpu: "4" + memory: 2Gi + diff --git a/test/performance/packages/complex-package/networkpolicy.yaml b/test/performance/packages/complex-package/networkpolicy.yaml new file mode 100644 index 000000000..90d0bb71b --- /dev/null +++ b/test/performance/packages/complex-package/networkpolicy.yaml @@ -0,0 +1,38 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +spec: + podSelector: + matchLabels: + app: complex-app # kpt-set: ${app-name} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app: complex-app # kpt-set: ${app-name} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default # kpt-set: ${namespace} + ports: + - protocol: TCP + port: 8080 # kpt-set: ${port} + egress: + - to: + - namespaceSelector: {} + ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 80 + diff --git a/test/performance/packages/complex-package/poddisruptionbudget.yaml b/test/performance/packages/complex-package/poddisruptionbudget.yaml new file mode 100644 index 000000000..8e2d9b8c7 --- /dev/null +++ b/test/performance/packages/complex-package/poddisruptionbudget.yaml @@ -0,0 +1,17 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +spec: + minAvailable: 1 + selector: + matchLabels: + app: complex-app # kpt-set: ${app-name} + diff --git a/test/performance/packages/complex-package/role.yaml b/test/performance/packages/complex-package/role.yaml new file mode 100644 index 000000000..27483eb4d --- /dev/null +++ b/test/performance/packages/complex-package/role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +rules: + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "update", "patch"] + diff --git a/test/performance/packages/complex-package/rolebinding.yaml b/test/performance/packages/complex-package/rolebinding.yaml new file mode 100644 index 000000000..36f5da2a4 --- /dev/null +++ b/test/performance/packages/complex-package/rolebinding.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +subjects: + - kind: ServiceAccount + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} +roleRef: + kind: Role + name: complex-app # kpt-set: ${app-name} + apiGroup: rbac.authorization.k8s.io + diff --git a/test/performance/packages/complex-package/service.yaml b/test/performance/packages/complex-package/service.yaml new file mode 100644 index 000000000..d162da843 --- /dev/null +++ b/test/performance/packages/complex-package/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} +spec: + selector: + app: complex-app # kpt-set: ${app-name} + ports: + - name: http + protocol: TCP + port: 8080 # kpt-set: ${port} + targetPort: 80 # kpt-set: ${target-port} + type: ClusterIP + diff --git a/test/performance/packages/complex-package/serviceaccount.yaml b/test/performance/packages/complex-package/serviceaccount.yaml new file mode 100644 index 000000000..f4d7409fc --- /dev/null +++ b/test/performance/packages/complex-package/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: complex-app # kpt-set: ${app-name} + namespace: default # kpt-set: ${namespace} + labels: + app: complex-app # kpt-set: ${app-name} + environment: dev # kpt-set: ${environment} + annotations: + app.kubernetes.io/managed-by: kpt + team: platform # kpt-set: ${team} + diff --git a/test/performance/packages/complex-package/starlark-run.yaml b/test/performance/packages/complex-package/starlark-run.yaml new file mode 100644 index 000000000..cd5e74a25 --- /dev/null +++ b/test/performance/packages/complex-package/starlark-run.yaml @@ -0,0 +1,23 @@ +apiVersion: fn.kpt.dev/v1alpha1 +kind: StarlarkRun +metadata: + name: add-kpt-processed-annotation + annotations: + config.kubernetes.io/local-config: "true" +source: | + # Add a kpt.dev/starlark-processed annotation to every workload resource + # and enforce that Deployments always have a non-zero replica count. + def process(resources): + for resource in resources: + if "metadata" not in resource: + continue + if "annotations" not in resource["metadata"]: + resource["metadata"]["annotations"] = {} + resource["metadata"]["annotations"]["kpt.dev/starlark-processed"] = "true" + # Ensure replicas >= 1 for all Deployments + if resource.get("kind") == "Deployment": + spec = resource.get("spec", {}) + if spec.get("replicas", 1) < 1: + resource["spec"]["replicas"] = 1 + process(ctx.resource_list["items"]) + diff --git a/test/performance/packages/large-package/Kptfile b/test/performance/packages/large-package/Kptfile new file mode 100644 index 000000000..eb8ee5577 --- /dev/null +++ b/test/performance/packages/large-package/Kptfile @@ -0,0 +1,129 @@ +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: perf-test-large-package + annotations: + config.kubernetes.io/local-config: "true" +info: + description: Large test package for Porch performance testing - 30 microservice Deployments, uses every kpt catalog function +pipeline: + mutators: + # 1. set-namespace — stamps the target namespace on every namespaced resource + - image: CHANGE_IMAGE/set-namespace:v0.4.1 + configMap: + namespace: CHANGE_NAMESPACE + # 2. create-setters — scans resources and inserts # kpt-set: ${name} setter comments + - image: CHANGE_IMAGE/create-setters:v0.1.0 + configMap: + app-name: large-app + environment: dev + image: docker.io/nginx:latest + init-image: docker.io/busybox:1.35 + envoy-image: docker.io/envoyproxy/envoy:v1.28.0 + fluentbit-image: docker.io/fluent/fluent-bit:2.2 + replicas: "2" + port: "8080" + team: platform + version: v1.0.0 + log-level: info + db-host: postgres-service + db-port: "5432" + db-name: appdb + redis-host: redis-service + redis-port: "6379" + kafka-brokers: kafka-service:9092 + tracing-endpoint: http://jaeger-collector:14268/api/traces + elasticsearch-host: elasticsearch-service + elasticsearch-port: "9200" + rate-limit: "100" + timeout: "30" + cpu-request: 100m + cpu-limit: 500m + memory-request: 128Mi + memory-limit: 256Mi + image-pull-secret: registry-credentials + # 3. apply-setters — substitutes all # kpt-set: ${name} markers with configured values + - image: CHANGE_IMAGE/apply-setters:v0.2.0 + configMap: + app-name: large-app + environment: dev + image: docker.io/nginx:latest + init-image: docker.io/busybox:1.35 + envoy-image: docker.io/envoyproxy/envoy:v1.28.0 + fluentbit-image: docker.io/fluent/fluent-bit:2.2 + replicas: "2" + port: "8080" + team: platform + version: v1.0.0 + log-level: info + db-host: postgres-service + db-port: "5432" + db-name: appdb + redis-host: redis-service + redis-port: "6379" + kafka-brokers: kafka-service:9092 + tracing-endpoint: http://jaeger-collector:14268/api/traces + elasticsearch-host: elasticsearch-service + elasticsearch-port: "9200" + rate-limit: "100" + timeout: "30" + cpu-request: 100m + cpu-limit: 500m + memory-request: 128Mi + memory-limit: 256Mi + image-pull-secret: registry-credentials + # 4. set-labels — bulk-stamps labels onto every resource + - image: CHANGE_IMAGE/set-labels:v0.2.0 + configMap: + app.kubernetes.io/part-of: large-app + managed-by: kpt + team: platform + # 5. set-annotations — bulk-stamps annotations onto every resource + - image: CHANGE_IMAGE/set-annotations:v0.1.4 + configMap: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + # 6. set-image — updates the nginx container image name and tag + - image: CHANGE_IMAGE/set-image:v0.1.1 + configMap: + name: nginx + newName: docker.io/nginx + newTag: latest + # 7. search-replace — finds a specific field path and overwrites its value + - image: CHANGE_IMAGE/search-replace:v0.2.0 + configMap: + by-path: metadata.annotations.[app.kubernetes.io/managed-by] + put-value: kpt + # 8. ensure-name-substring — prepends 'large-' to any resource name that + # does not already start with that prefix + - image: CHANGE_IMAGE/ensure-name-substring:v0.1.1 + configMap: + prepend: large- + selectors: + - kind: Deployment + - kind: LimitRange + # 9. apply-replacements — propagates field values across resources as defined + # in the ApplyReplacements resource inside the package + - image: CHANGE_IMAGE/apply-replacements:v0.1.1 + configPath: apply-replacements.yaml + # 10. starlark — executes the StarlarkRun script from the package + - image: CHANGE_IMAGE/starlark:v0.4.3 + configPath: starlark-run.yaml + # 11. upsert-resource — inserts or updates the LimitRange into the package + - image: CHANGE_IMAGE/upsert-resource:v0.2.0 + configPath: limitrange.yaml + # 12. generate-folders — generates Config Controller Folder resources; no-op + # when no HierarchyConfig resources are present in the package + - image: CHANGE_IMAGE/generate-folders:v0.1.1 + # 13. set-enforcement-action — sets enforcementAction: warn on all Constraint + # resources (defined in gatekeeper-constraints.yaml) + - image: CHANGE_IMAGE/set-enforcement-action:v0.1.1 + configMap: + enforcementAction: warn + validators: + # 14. kubeconform — validates KRM resources against their JSON schemas + # (uses baked-in schemas; see https://catalog.kpt.dev/kubeconform/v0.1/) + - image: CHANGE_IMAGE/kubeconform:v0.1.1 + # 15. gatekeeper — validates resources against ConstraintTemplate/Constraint + # policies defined in gatekeeper-constraints.yaml + - image: CHANGE_IMAGE/gatekeeper:v0.2.1 diff --git a/test/performance/packages/large-package/apply-replacements.yaml b/test/performance/packages/large-package/apply-replacements.yaml new file mode 100644 index 000000000..d5fd82387 --- /dev/null +++ b/test/performance/packages/large-package/apply-replacements.yaml @@ -0,0 +1,39 @@ +apiVersion: fn.kpt.dev/v1alpha1 +kind: ApplyReplacements +metadata: + name: apply-replacements + annotations: + config.kubernetes.io/local-config: "true" +replacements: + # Copy the db-host value from api-gateway into every Deployment that declares DB_HOST + # (demonstrates cross-resource field propagation; runs after ensure-name-substring) + - source: + kind: Deployment + name: large-api-gateway + fieldPath: spec.template.spec.containers.[name=api-gateway].env.[name=DB_HOST].value + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.spec.containers.*.env.[name=DB_HOST].value + # Propagate the kafka brokers value consistently across all Deployments + - source: + kind: Deployment + name: large-api-gateway + fieldPath: spec.template.spec.containers.[name=api-gateway].env.[name=KAFKA_BROKERS].value + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.spec.containers.*.env.[name=KAFKA_BROKERS].value + # Propagate the tracing endpoint value consistently across all Deployments + - source: + kind: Deployment + name: large-api-gateway + fieldPath: spec.template.spec.containers.[name=api-gateway].env.[name=TRACING_ENDPOINT].value + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.spec.containers.*.env.[name=TRACING_ENDPOINT].value + diff --git a/test/performance/packages/large-package/deployment.yaml b/test/performance/packages/large-package/deployment.yaml new file mode 100644 index 000000000..ee33f0044 --- /dev/null +++ b/test/performance/packages/large-package/deployment.yaml @@ -0,0 +1,17220 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-gateway + namespace: default # kpt-set: ${namespace} + labels: + app: api-gateway + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: api-gateway + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for api-gateway +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: api-gateway + template: + metadata: + labels: + app: api-gateway + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - api-gateway + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: api-gateway + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing api-gateway..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for api-gateway..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: api-gateway + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: api-gateway + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: api-gateway-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: api-gateway + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: api-gateway + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: api-gateway-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: api-gateway + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: api-gateway-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: auth-service + namespace: default # kpt-set: ${namespace} + labels: + app: auth-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: auth-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for auth-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: auth-service + template: + metadata: + labels: + app: auth-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - auth-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: auth-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing auth-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for auth-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: auth-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: auth-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: auth-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: auth-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: auth-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: auth-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: auth-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: auth-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: user-service + namespace: default # kpt-set: ${namespace} + labels: + app: user-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: user-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for user-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: user-service + template: + metadata: + labels: + app: user-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - user-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: user-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing user-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for user-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: user-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: user-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: user-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: user-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: user-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: user-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: user-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: user-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: product-service + namespace: default # kpt-set: ${namespace} + labels: + app: product-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: product-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for product-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: product-service + template: + metadata: + labels: + app: product-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - product-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: product-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing product-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for product-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: product-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: product-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: product-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: product-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: product-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: product-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: product-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: product-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cart-service + namespace: default # kpt-set: ${namespace} + labels: + app: cart-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: cart-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for cart-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: cart-service + template: + metadata: + labels: + app: cart-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - cart-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: cart-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing cart-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for cart-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: cart-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: cart-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: cart-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: cart-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: cart-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: cart-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: cart-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: cart-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order-service + namespace: default # kpt-set: ${namespace} + labels: + app: order-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: order-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for order-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - order-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: order-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing order-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for order-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: order-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: order-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: order-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: order-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: order-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: order-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: order-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: order-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: default # kpt-set: ${namespace} + labels: + app: payment-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: payment-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for payment-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - payment-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: payment-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing payment-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for payment-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: payment-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: payment-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: payment-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: payment-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: payment-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: payment-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: payment-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: payment-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: notification-service + namespace: default # kpt-set: ${namespace} + labels: + app: notification-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: notification-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for notification-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: notification-service + template: + metadata: + labels: + app: notification-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - notification-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: notification-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing notification-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for notification-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: notification-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: notification-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: notification-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: notification-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: notification-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: notification-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: notification-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: notification-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inventory-service + namespace: default # kpt-set: ${namespace} + labels: + app: inventory-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: inventory-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for inventory-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: inventory-service + template: + metadata: + labels: + app: inventory-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - inventory-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inventory-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing inventory-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for inventory-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: inventory-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: inventory-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: inventory-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: inventory-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: inventory-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: inventory-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: inventory-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: inventory-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shipping-service + namespace: default # kpt-set: ${namespace} + labels: + app: shipping-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: shipping-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for shipping-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: shipping-service + template: + metadata: + labels: + app: shipping-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - shipping-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: shipping-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing shipping-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for shipping-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: shipping-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: shipping-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: shipping-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: shipping-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: shipping-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: shipping-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: shipping-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: shipping-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: review-service + namespace: default # kpt-set: ${namespace} + labels: + app: review-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: review-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for review-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: review-service + template: + metadata: + labels: + app: review-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - review-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: review-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing review-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for review-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: review-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: review-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: review-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: review-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: review-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: review-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: review-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: review-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: search-service + namespace: default # kpt-set: ${namespace} + labels: + app: search-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: search-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for search-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: search-service + template: + metadata: + labels: + app: search-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - search-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: search-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing search-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for search-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: search-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: search-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: search-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: search-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: search-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: search-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: search-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: search-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: recommendation-service + namespace: default # kpt-set: ${namespace} + labels: + app: recommendation-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: recommendation-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for recommendation-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: recommendation-service + template: + metadata: + labels: + app: recommendation-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - recommendation-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: recommendation-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing recommendation-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for recommendation-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: recommendation-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: recommendation-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: recommendation-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: recommendation-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: recommendation-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: recommendation-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: recommendation-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: recommendation-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: analytics-service + namespace: default # kpt-set: ${namespace} + labels: + app: analytics-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: analytics-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for analytics-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: analytics-service + template: + metadata: + labels: + app: analytics-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - analytics-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: analytics-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing analytics-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for analytics-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: analytics-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: analytics-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: analytics-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: analytics-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: analytics-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: analytics-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: analytics-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: analytics-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reporting-service + namespace: default # kpt-set: ${namespace} + labels: + app: reporting-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: reporting-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for reporting-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: reporting-service + template: + metadata: + labels: + app: reporting-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - reporting-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: reporting-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing reporting-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for reporting-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: reporting-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: reporting-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: reporting-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: reporting-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: reporting-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: reporting-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: reporting-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: reporting-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-service + namespace: default # kpt-set: ${namespace} + labels: + app: admin-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: admin-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for admin-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: admin-service + template: + metadata: + labels: + app: admin-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - admin-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: admin-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing admin-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for admin-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: admin-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: admin-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: admin-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: admin-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: admin-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: admin-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: admin-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: admin-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webhook-service + namespace: default # kpt-set: ${namespace} + labels: + app: webhook-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: webhook-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for webhook-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: webhook-service + template: + metadata: + labels: + app: webhook-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - webhook-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: webhook-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing webhook-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for webhook-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: webhook-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: webhook-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: webhook-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: webhook-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: webhook-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: webhook-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: webhook-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: webhook-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: job-scheduler + namespace: default # kpt-set: ${namespace} + labels: + app: job-scheduler + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: job-scheduler + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for job-scheduler +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: job-scheduler + template: + metadata: + labels: + app: job-scheduler + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - job-scheduler + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: job-scheduler + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing job-scheduler..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for job-scheduler..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: job-scheduler + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: job-scheduler + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: job-scheduler-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: job-scheduler + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: job-scheduler + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: job-scheduler-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: job-scheduler + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: job-scheduler-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cache-service + namespace: default # kpt-set: ${namespace} + labels: + app: cache-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: cache-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for cache-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: cache-service + template: + metadata: + labels: + app: cache-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - cache-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: cache-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing cache-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for cache-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: cache-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: cache-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: cache-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: cache-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: cache-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: cache-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: cache-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: cache-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: media-service + namespace: default # kpt-set: ${namespace} + labels: + app: media-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: media-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for media-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: media-service + template: + metadata: + labels: + app: media-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - media-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: media-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing media-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for media-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: media-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: media-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: media-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: media-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: media-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: media-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: media-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: media-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: email-service + namespace: default # kpt-set: ${namespace} + labels: + app: email-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: email-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for email-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: email-service + template: + metadata: + labels: + app: email-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - email-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: email-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing email-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for email-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: email-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: email-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: email-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: email-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: email-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: email-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: email-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: email-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sms-service + namespace: default # kpt-set: ${namespace} + labels: + app: sms-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: sms-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for sms-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: sms-service + template: + metadata: + labels: + app: sms-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - sms-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: sms-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing sms-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for sms-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: sms-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: sms-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: sms-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: sms-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: sms-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: sms-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: sms-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: sms-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: audit-service + namespace: default # kpt-set: ${namespace} + labels: + app: audit-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: audit-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for audit-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: audit-service + template: + metadata: + labels: + app: audit-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - audit-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: audit-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing audit-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for audit-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: audit-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: audit-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: audit-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: audit-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: audit-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: audit-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: audit-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: audit-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: config-service + namespace: default # kpt-set: ${namespace} + labels: + app: config-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: config-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for config-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: config-service + template: + metadata: + labels: + app: config-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - config-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: config-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing config-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for config-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: config-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: config-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: config-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: config-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: config-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: config-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: config-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: config-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: discovery-service + namespace: default # kpt-set: ${namespace} + labels: + app: discovery-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: discovery-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for discovery-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: discovery-service + template: + metadata: + labels: + app: discovery-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - discovery-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: discovery-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing discovery-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for discovery-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: discovery-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: discovery-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: discovery-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: discovery-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: discovery-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: discovery-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: discovery-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: discovery-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-collector + namespace: default # kpt-set: ${namespace} + labels: + app: metrics-collector + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: metrics-collector + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for metrics-collector +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: metrics-collector + template: + metadata: + labels: + app: metrics-collector + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - metrics-collector + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: metrics-collector + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing metrics-collector..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for metrics-collector..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: metrics-collector + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: metrics-collector + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: metrics-collector-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: metrics-collector + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: metrics-collector + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: metrics-collector-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: metrics-collector + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: metrics-collector-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: log-aggregator + namespace: default # kpt-set: ${namespace} + labels: + app: log-aggregator + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: log-aggregator + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for log-aggregator +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: log-aggregator + template: + metadata: + labels: + app: log-aggregator + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - log-aggregator + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: log-aggregator + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing log-aggregator..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for log-aggregator..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: log-aggregator + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: log-aggregator + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: log-aggregator-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: log-aggregator + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: log-aggregator + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: log-aggregator-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: log-aggregator + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: log-aggregator-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trace-collector + namespace: default # kpt-set: ${namespace} + labels: + app: trace-collector + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: trace-collector + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for trace-collector +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: trace-collector + template: + metadata: + labels: + app: trace-collector + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - trace-collector + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: trace-collector + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing trace-collector..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for trace-collector..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: trace-collector + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: trace-collector + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: trace-collector-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: trace-collector + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: trace-collector + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: trace-collector-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: trace-collector + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: trace-collector-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alerting-service + namespace: default # kpt-set: ${namespace} + labels: + app: alerting-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: alerting-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for alerting-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: alerting-service + template: + metadata: + labels: + app: alerting-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - alerting-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: alerting-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing alerting-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for alerting-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: alerting-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: alerting-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: alerting-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: alerting-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: alerting-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: alerting-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: alerting-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: alerting-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dashboard-service + namespace: default # kpt-set: ${namespace} + labels: + app: dashboard-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + managed-by: kpt + app.kubernetes.io/name: dashboard-service + app.kubernetes.io/component: microservice + app.kubernetes.io/part-of: large-app + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app + app.kubernetes.io/version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + deployment.kubernetes.io/revision: "1" + description: Microservice deployment for dashboard-service +spec: + replicas: 2 # kpt-set: ${replicas} + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: dashboard-service + template: + metadata: + labels: + app: dashboard-service + environment: dev # kpt-set: ${environment} + version: v1.0.0 # kpt-set: ${version} + team: platform # kpt-set: ${team} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: /metrics + spec: + serviceAccountName: large-app # kpt-set: ${app-name} + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - dashboard-service + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-type + operator: In + values: + - application + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: dashboard-service + initContainers: + - name: init-config + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Initializing dashboard-service..." + echo "Environment: $APP_ENV" + echo "Version: $APP_VERSION" + until nc -z $DB_HOST $DB_PORT; do + echo "Waiting for database at $DB_HOST:$DB_PORT..." + sleep 2 + done + echo "Database is reachable. Init complete." + env: + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: init-migrations + image: docker.io/busybox:1.35 # kpt-set: ${init-image} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo "Running schema migrations for dashboard-service..." + env: + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp-dir + mountPath: /tmp + containers: + - name: dashboard-service + image: docker.io/nginx:latest # kpt-set: ${image} + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 # kpt-set: ${port} + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + - name: health + containerPort: 8081 + protocol: TCP + - name: grpc + containerPort: 9000 + protocol: TCP + - name: admin + containerPort: 8082 + protocol: TCP + env: + - name: APP_NAME + value: dashboard-service + - name: APP_ENV + value: dev # kpt-set: ${environment} + - name: APP_VERSION + value: v1.0.0 # kpt-set: ${version} + - name: APP_PORT + value: "8080" # kpt-set: ${port} + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: LOG_FORMAT + value: json + - name: METRICS_PORT + value: "9090" + - name: HEALTH_PORT + value: "8081" + - name: GRPC_PORT + value: "9000" + - name: ADMIN_PORT + value: "8082" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DB_HOST + value: postgres-service # kpt-set: ${db-host} + - name: DB_PORT + value: "5432" # kpt-set: ${db-port} + - name: DB_NAME + value: appdb # kpt-set: ${db-name} + - name: DB_USER + valueFrom: + secretKeyRef: + name: db-credentials + key: username + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: DB_SSL_MODE + value: require + - name: DB_MAX_OPEN_CONNS + value: "25" + - name: DB_MAX_IDLE_CONNS + value: "5" + - name: DB_CONN_MAX_LIFETIME + value: 5m + - name: REDIS_HOST + value: redis-service # kpt-set: ${redis-host} + - name: REDIS_PORT + value: "6379" # kpt-set: ${redis-port} + - name: REDIS_DB + value: "0" + - name: REDIS_POOL_SIZE + value: "10" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-credentials + key: password + - name: KAFKA_BROKERS + value: kafka-service:9092 # kpt-set: ${kafka-brokers} + - name: KAFKA_GROUP_ID + value: dashboard-service-group + - name: KAFKA_AUTO_OFFSET_RESET + value: earliest + - name: KAFKA_MAX_POLL_RECORDS + value: "500" + - name: TRACING_ENDPOINT + value: http://jaeger-collector:14268/api/traces # kpt-set: ${tracing-endpoint} + - name: TRACING_ENABLED + value: "true" + - name: TRACING_SAMPLE_RATE + value: "0.1" + - name: TRACING_SERVICE_NAME + value: dashboard-service + - name: METRICS_ENABLED + value: "true" + - name: PROFILING_ENABLED + value: "false" + - name: RATE_LIMIT_RPS + value: "100" # kpt-set: ${rate-limit} + - name: TIMEOUT_SECONDS + value: "30" # kpt-set: ${timeout} + - name: MAX_CONNECTIONS + value: "100" + - name: TLS_ENABLED + value: "true" + - name: TLS_CERT_FILE + value: /etc/tls/tls.crt + - name: TLS_KEY_FILE + value: /etc/tls/tls.key + - name: CORS_ORIGINS + value: "*" + - name: CORS_METHODS + value: GET,POST,PUT,DELETE,OPTIONS + - name: CORS_HEADERS + value: Content-Type,Authorization,X-Request-ID + - name: JWT_ISSUER + value: large-app # kpt-set: ${app-name} + - name: JWT_AUDIENCE + value: large-app-users + - name: JWT_EXPIRY + value: 3600s + - name: AUTH_SERVICE_URL + value: http://auth-service:8080 + - name: CONFIG_SERVICE_URL + value: http://config-service:8080 + - name: DISCOVERY_SERVICE_URL + value: http://discovery-service:8080 + - name: FEATURE_FLAGS_ENABLED + value: "true" + - name: CACHE_TTL_SECONDS + value: "300" + - name: SHUTDOWN_TIMEOUT_SECONDS + value: "30" + - name: MAX_REQUEST_BODY_SIZE + value: "10485760" + - name: GZIP_ENABLED + value: "true" + - name: KEEP_ALIVE_TIMEOUT + value: "75s" + - name: READ_TIMEOUT + value: "30s" + - name: WRITE_TIMEOUT + value: "30s" + - name: IDLE_TIMEOUT + value: "120s" + resources: + requests: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + limits: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: liveness + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /startupz + port: 8081 + httpHeaders: + - name: X-Health-Check + value: startup + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: logs-dir + mountPath: /var/log/app + - name: tls-certs + mountPath: /etc/tls + readOnly: true + - name: feature-flags + mountPath: /etc/feature-flags + readOnly: true + - name: envoy-proxy + image: docker.io/envoyproxy/envoy:v1.28.0 # kpt-set: ${envoy-image} + imagePullPolicy: IfNotPresent + ports: + - name: envoy-admin + containerPort: 9901 + protocol: TCP + - name: envoy-inbound + containerPort: 15001 + protocol: TCP + - name: envoy-outbound + containerPort: 15002 + protocol: TCP + args: + - -c + - /etc/envoy/envoy.yaml + - --log-level + - info # kpt-set: ${log-level} + - --drain-time-s + - "30" + env: + - name: ENVOY_UID + value: "0" + - name: ENVOY_ADMIN_PORT + value: "9901" + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVICE_NAME + value: dashboard-service + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /healthcheck/fail + port: 9901 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthcheck/ok + port: 9901 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + readOnly: true + - name: tmp-dir + mountPath: /tmp + - name: log-shipper + image: docker.io/fluent/fluent-bit:2.2 # kpt-set: ${fluentbit-image} + imagePullPolicy: IfNotPresent + ports: + - name: fluent-http + containerPort: 2020 + protocol: TCP + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch-service # kpt-set: ${elasticsearch-host} + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" # kpt-set: ${elasticsearch-port} + - name: FLUENT_ELASTICSEARCH_INDEX + value: dashboard-service-logs + - name: FLUENT_ELASTICSEARCH_TYPE + value: _doc + - name: LOG_LEVEL + value: info # kpt-set: ${log-level} + - name: SERVICE_NAME + value: dashboard-service + - name: ENVIRONMENT + value: dev # kpt-set: ${environment} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + livenessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/v1/health + port: 2020 + initialDelaySeconds: 5 + periodSeconds: 15 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: logs-dir + mountPath: /var/log/app + readOnly: true + - name: fluent-bit-config + mountPath: /fluent-bit/etc + readOnly: true + - name: tmp-dir + mountPath: /tmp + volumes: + - name: config-volume + configMap: + name: large-app-config # kpt-set: ${app-name}-config + defaultMode: 0440 + - name: envoy-config + configMap: + name: envoy-config + defaultMode: 0440 + - name: fluent-bit-config + configMap: + name: fluent-bit-config + defaultMode: 0440 + - name: feature-flags + configMap: + name: feature-flags-config + defaultMode: 0440 + - name: tls-certs + secret: + secretName: dashboard-service-tls + defaultMode: 0400 + - name: tmp-dir + emptyDir: + medium: Memory + sizeLimit: 64Mi + - name: logs-dir + emptyDir: + sizeLimit: 512Mi + imagePullSecrets: + - name: registry-credentials # kpt-set: ${image-pull-secret} + dnsConfig: + options: + - name: ndots + value: "2" + - name: edns0 + restartPolicy: Always diff --git a/test/performance/packages/large-package/gatekeeper-constraints.yaml b/test/performance/packages/large-package/gatekeeper-constraints.yaml new file mode 100644 index 000000000..c72188b7d --- /dev/null +++ b/test/performance/packages/large-package/gatekeeper-constraints.yaml @@ -0,0 +1,53 @@ +--- +# ConstraintTemplate: OPA Rego policy that checks for required labels. +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: requiredlabels + annotations: + config.kubernetes.io/local-config: "true" +spec: + crd: + spec: + names: + kind: RequiredLabels + validation: + openAPIV3Schema: + type: object + properties: + labels: + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package requiredlabels + + violation[{"msg": msg, "details": {"missing_labels": missing}}] { + provided := {label | input.review.object.metadata.labels[label]} + required := {label | label := input.parameters.labels[_]} + missing := required - provided + count(missing) > 0 + msg := sprintf("you must provide labels: %v", [missing]) + } +--- +# Constraint: enforces that all Deployments carry the required labels. +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: RequiredLabels +metadata: + name: must-have-required-labels + annotations: + config.kubernetes.io/local-config: "true" +spec: + enforcementAction: warn + match: + kinds: + - apiGroups: ["apps"] + kinds: ["Deployment"] + parameters: + labels: + - app + - environment + - team + diff --git a/test/performance/packages/large-package/limitrange.yaml b/test/performance/packages/large-package/limitrange.yaml new file mode 100644 index 000000000..428a58112 --- /dev/null +++ b/test/performance/packages/large-package/limitrange.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: LimitRange +metadata: + name: large-app-limits + namespace: default # kpt-set: ${namespace} + labels: + app.kubernetes.io/part-of: large-app + managed-by: kpt + team: platform # kpt-set: ${team} + annotations: + app.kubernetes.io/managed-by: kpt + app.kubernetes.io/part-of: large-app +spec: + limits: + - type: Container + default: + cpu: 500m # kpt-set: ${cpu-limit} + memory: 256Mi # kpt-set: ${memory-limit} + defaultRequest: + cpu: 100m # kpt-set: ${cpu-request} + memory: 128Mi # kpt-set: ${memory-request} + max: + cpu: "2" + memory: 1Gi + min: + cpu: 10m + memory: 16Mi + - type: Pod + max: + cpu: "8" + memory: 4Gi + - type: PersistentVolumeClaim + max: + storage: 10Gi + min: + storage: 1Gi + diff --git a/test/performance/packages/large-package/starlark-run.yaml b/test/performance/packages/large-package/starlark-run.yaml new file mode 100644 index 000000000..ebf47d292 --- /dev/null +++ b/test/performance/packages/large-package/starlark-run.yaml @@ -0,0 +1,38 @@ +apiVersion: fn.kpt.dev/v1alpha1 +kind: StarlarkRun +metadata: + name: add-kpt-processed-annotation + annotations: + config.kubernetes.io/local-config: "true" +source: | + # Add a kpt.dev/starlark-processed annotation to every workload resource, + # enforce replicas >= 1 on all Deployments, and verify that every container + # declares resource limits. + def process(resources): + for resource in resources: + if "metadata" not in resource: + continue + if "annotations" not in resource["metadata"]: + resource["metadata"]["annotations"] = {} + resource["metadata"]["annotations"]["kpt.dev/starlark-processed"] = "true" + + if resource.get("kind") != "Deployment": + continue + + spec = resource.get("spec", {}) + # Ensure replicas is at least 1 + if spec.get("replicas", 1) < 1: + resource["spec"]["replicas"] = 1 + + # Ensure every container has resource limits set + containers = spec.get("template", {}).get("spec", {}).get("containers", []) + for container in containers: + if "resources" not in container: + container["resources"] = {} + if "limits" not in container["resources"]: + container["resources"]["limits"] = {"cpu": "500m", "memory": "256Mi"} + if "requests" not in container["resources"]: + container["resources"]["requests"] = {"cpu": "100m", "memory": "128Mi"} + + process(ctx.resource_list["items"]) + diff --git a/test/performance/packages/small-package/Kptfile b/test/performance/packages/small-package/Kptfile new file mode 100644 index 000000000..88785e51a --- /dev/null +++ b/test/performance/packages/small-package/Kptfile @@ -0,0 +1,16 @@ +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: perf-test-package + annotations: + config.kubernetes.io/local-config: "true" +info: + description: Generic test package for Porch performance testing +pipeline: + mutators: + - image: CHANGE_IMAGE/set-namespace:v0.4.1 + configMap: + namespace: CHANGE_NAMESPACE + - image: CHANGE_IMAGE/apply-setters:v0.2.0 + configMap: + image: docker.io/nginx:latest diff --git a/test/performance/packages/small-package/deployment.yaml b/test/performance/packages/small-package/deployment.yaml new file mode 100644 index 000000000..385585738 --- /dev/null +++ b/test/performance/packages/small-package/deployment.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment + namespace: default # kpt-set: ${namespace} + labels: + app: test-app +spec: + replicas: 1 + selector: + matchLabels: + app: test-app + template: + metadata: + labels: + app: test-app + spec: + containers: + - name: test-container + image: nginx:latest # kpt-set: ${image} diff --git a/test/performance/performance_suite.go b/test/performance/performance_suite.go index e48368ea5..ffdd0fd1a 100644 --- a/test/performance/performance_suite.go +++ b/test/performance/performance_suite.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The Nephio Authors +// Copyright 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -64,14 +64,13 @@ var ( enableDeletion = flag.Bool("enable-deletion", false, "Enable deletion of package revisions at the end of the test") enablePrometheus = flag.Bool("enable-prometheus", false, "Enable Prometheus metrics server on port 9091") - metricsLogFile = flag.String("metrics-log-prefix", "porch-metrics", "Prefix for the timestamped metrics log file") - resultsFile = flag.String("results-file", "load_test_results.txt", "File name for test results") - fullLogFile = flag.String("detailed-log-file", "load_test.log", "File name for detailed log") - lifecycleCSV = flag.String("repo-results-csv", "load_test_lifecycle_results.csv", "File name for repository results CSV") - operationsCSV = flag.String("operations-csv", "load_test_operations_results.csv", "File name for operations details CSV") - deletionCSV = flag.String("deletion-csv", "load_test_deletion_results.csv", "File name for deletion operations CSV") - kptfilePath = flag.String("kptfile-path", "resources/Kptfile", "Path to the Kptfile") - packageResourcesPath = flag.String("package-resources-path", "resources/deployment.yaml", "Path to the package resources") + metricsLogFile = flag.String("metrics-log-prefix", "porch-metrics", "Prefix for the timestamped metrics log file") + resultsFile = flag.String("results-file", "load_test_results.txt", "File name for test results") + fullLogFile = flag.String("detailed-log-file", "load_test.log", "File name for detailed log") + lifecycleCSV = flag.String("repo-results-csv", "load_test_lifecycle_results.csv", "File name for repository results CSV") + operationsCSV = flag.String("operations-csv", "load_test_operations_results.csv", "File name for operations details CSV") + deletionCSV = flag.String("deletion-csv", "load_test_deletion_results.csv", "File name for deletion operations CSV") + packagePath = flag.String("package-path", "packages/large-package", "Path to the directory containing package resources") giteaURL = flag.String("gitea-url", "http://localhost:3000", "Base URL for the Gitea API") giteaUsername = flag.String("gitea-username", "porch", "Gitea username") @@ -108,21 +107,20 @@ type PerfTestSuite struct { } type TestOptions struct { - namespace string - numRepos int - numPkgs int - numRevs int - repoParallelism int - packageParallelism int - errorRate float64 - enableDeletion bool - kptfilePath string - packageResourcesPath string - krmFnRegistryURL string - giteaURL string - giteaUsername string - giteaPassword string - paddingSize int + namespace string + numRepos int + numPkgs int + numRevs int + repoParallelism int + packageParallelism int + errorRate float64 + enableDeletion bool + packagePath string + krmFnRegistryURL string + giteaURL string + giteaUsername string + giteaPassword string + paddingSize int } type LogOptions struct { @@ -208,21 +206,20 @@ func (t *PerfTestSuite) SetupSuite() { t.metrics = make(map[string]TestMetrics) t.testOptions = TestOptions{ - namespace: *namespace, - numRepos: *numRepos, - numPkgs: *numPackages, - numRevs: *numRevisions, - repoParallelism: *repoParallelism, - packageParallelism: *packageParallelism, - errorRate: *errorRate, - enableDeletion: *enableDeletion, - kptfilePath: *kptfilePath, - packageResourcesPath: *packageResourcesPath, - krmFnRegistryURL: getEnvWithDefault("PORCH_GHCR_PREFIX_URL", "ghcr.io/kptdev/krm-functions-catalog"), - giteaURL: *giteaURL, - giteaUsername: *giteaUsername, - giteaPassword: *giteaPassword, - paddingSize: *paddingSize, + namespace: *namespace, + numRepos: *numRepos, + numPkgs: *numPackages, + numRevs: *numRevisions, + repoParallelism: *repoParallelism, + packageParallelism: *packageParallelism, + errorRate: *errorRate, + enableDeletion: *enableDeletion, + packagePath: *packagePath, + krmFnRegistryURL: getEnvWithDefault("PORCH_GHCR_PREFIX_URL", "gcr.io/kptdev/krm-functions-catalog"), + giteaURL: *giteaURL, + giteaUsername: *giteaUsername, + giteaPassword: *giteaPassword, + paddingSize: *paddingSize, } t.logOptions = LogOptions{ @@ -765,7 +762,7 @@ func (t *PerfTestSuite) updateOrCreatePackageRevisionResources(repoName, pkgName return err } - pkgResources := t.createPackageResources(pkgRevName) + pkgResources := t.createPackageResources() if resources.Spec.Resources == nil { resources.Spec.Resources = make(map[string]string) } @@ -796,26 +793,40 @@ func (t *PerfTestSuite) updateOrCreatePackageRevisionResources(repoName, pkgName return nil } -func (t *PerfTestSuite) createPackageResources(pkgName string) map[string]string { - resources := make(map[string]string) - - resources["Kptfile"] = t.readResourcesFromDir(t.testOptions.kptfilePath) - resources["deployment.yaml"] = t.readResourcesFromDir(t.testOptions.packageResourcesPath) - - resources["Kptfile"] = strings.ReplaceAll(resources["Kptfile"], "CHANGE_ME", pkgName) - resources["Kptfile"] = strings.ReplaceAll(resources["Kptfile"], "REGISTRY_URL", t.testOptions.krmFnRegistryURL) - resources["deployment.yaml"] = strings.ReplaceAll(resources["deployment.yaml"], "CHANGE_ME", pkgName) - - if t.testOptions.paddingSize > 0 { - resources["largefile.yaml"] = fmt.Sprintf(`apiVersion: v1 -kind: ConfigMap -metadata: - name: padding-data -data: - value: "%s" -`, strings.Repeat("a", t.testOptions.paddingSize*1024*1024)) +func (t *PerfTestSuite) createPackageResources() map[string]string { + resources := t.readPackageResources(t.testOptions.packagePath) + if kptfile, ok := resources["Kptfile"]; ok { + kptfile = strings.ReplaceAll(kptfile, "CHANGE_NAMESPACE", t.testOptions.namespace) + kptfile = strings.ReplaceAll(kptfile, "CHANGE_IMAGE", t.testOptions.krmFnRegistryURL) + resources["Kptfile"] = kptfile } + return resources +} +func (t *PerfTestSuite) readPackageResources(dir string) map[string]string { + t.T().Helper() + resources := make(map[string]string) + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + content, readErr := os.ReadFile(path) + if readErr != nil { + t.T().Fatalf("ReadFile(%q) failed: %v", path, readErr) + } + relPath, relErr := filepath.Rel(dir, path) + if relErr != nil { + return relErr + } + resources[relPath] = string(content) + return nil + }) + if err != nil { + t.T().Fatalf("WalkDir(%s) failed: %v", dir, err) + } return resources } @@ -972,24 +983,3 @@ func (t *PerfTestSuite) deletePackageRevision(repoName, pkgName, pkgRevName stri return nil } - -func (t *PerfTestSuite) readResourcesFromDir(dir string) string { - t.T().Helper() - var content []byte - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - content, err = os.ReadFile(path) - if err != nil { - t.T().Fatalf("ReadFile(%q) failed: %v", path, err) - } - } - return nil - }) - if err != nil { - t.T().Fatalf("WalkDir(%s) failed: %v", dir, err) - } - return string(content) -} diff --git a/test/performance/porch_metrics_test.go b/test/performance/porch_metrics_test.go index 6b27ce1ed..eac9670ed 100644 --- a/test/performance/porch_metrics_test.go +++ b/test/performance/porch_metrics_test.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The Nephio Authors +// Copyright 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/performance/resources/Kptfile b/test/performance/resources/Kptfile deleted file mode 100644 index 0464fddca..000000000 --- a/test/performance/resources/Kptfile +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kpt.dev/v1 -kind: Kptfile -metadata: - name: CHANGE_ME - annotations: - config.kubernetes.io/local-config: "true" -info: - description: network function CHANGE_ME blueprint -pipeline: - mutators: - - image: REGISTRY_URL/set-namespace:v0.4.1 - configMap: - namespace: CHANGE_ME \ No newline at end of file diff --git a/test/performance/resources/deployment.yaml b/test/performance/resources/deployment.yaml deleted file mode 100644 index 66c3fe5fa..000000000 --- a/test/performance/resources/deployment.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: CHANGE_ME -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: CHANGE_ME - template: - metadata: - labels: - app.kubernetes.io/name: CHANGE_ME - spec: - containers: - - name: nginx - image: nginx:latest \ No newline at end of file diff --git a/test/performance/types.go b/test/performance/types.go index 0c07fa027..0ff3850de 100644 --- a/test/performance/types.go +++ b/test/performance/types.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Nephio Authors +// Copyright 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From f136e03405c35a9aa1b38b77156b4fc2a9c7bcdd Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Sat, 18 Jul 2026 01:18:16 +0200 Subject: [PATCH 03/13] Allow for perf tests to work with both v1alpha1 and v1alpha2 Signed-off-by: Rendre Greyling --- .../grafana-package-sizes-dashboard.json | 2 +- .../grafana-perf-test-dashboard.json | 612 +++++++++++++++--- internal/telemetry/metrics.go | 16 +- test/performance/csv_generation.go | 51 +- test/performance/driver.go | 349 ++++++++++ test/performance/driver_v1alpha1.go | 180 ++++++ test/performance/driver_v1alpha2.go | 322 +++++++++ test/performance/logger.go | 20 +- test/performance/operations.go | 108 ++++ test/performance/performance_suite.go | 392 ++--------- test/performance/porch_metrics_test.go | 135 ++-- test/performance/types.go | 46 +- 12 files changed, 1653 insertions(+), 580 deletions(-) create mode 100644 test/performance/driver.go create mode 100644 test/performance/driver_v1alpha1.go create mode 100644 test/performance/driver_v1alpha2.go create mode 100644 test/performance/operations.go diff --git a/deployments/metrics-resources/grafana-package-sizes-dashboard.json b/deployments/metrics-resources/grafana-package-sizes-dashboard.json index 68aebf826..e98249e2f 100644 --- a/deployments/metrics-resources/grafana-package-sizes-dashboard.json +++ b/deployments/metrics-resources/grafana-package-sizes-dashboard.json @@ -18,7 +18,7 @@ }, "overrides": [] }, - "gridPos": { "h": 100, "w": 24, "x": 0, "y": 0 }, + "gridPos": { "h": 25, "w": 24, "x": 0, "y": 0 }, "id": 101, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ diff --git a/deployments/metrics-resources/grafana-perf-test-dashboard.json b/deployments/metrics-resources/grafana-perf-test-dashboard.json index c986ce3ff..6fc7e0050 100644 --- a/deployments/metrics-resources/grafana-perf-test-dashboard.json +++ b/deployments/metrics-resources/grafana-perf-test-dashboard.json @@ -30,7 +30,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GITEA-REPO-CREATE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GITEA-REPO-CREATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -39,7 +39,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GITEA-REPO-CREATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GITEA-REPO-CREATE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GITEA-REPO-CREATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GITEA-REPO-CREATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -48,7 +48,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GITEA-REPO-CREATE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GITEA-REPO-CREATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -108,7 +108,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -116,7 +116,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"GITEA-REPO-CREATE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -173,7 +173,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PORCH-REPO-CREATE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PORCH-REPO-CREATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -182,7 +182,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PORCH-REPO-CREATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PORCH-REPO-CREATE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PORCH-REPO-CREATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PORCH-REPO-CREATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -191,7 +191,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PORCH-REPO-CREATE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PORCH-REPO-CREATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -250,7 +250,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -258,7 +258,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"PORCH-REPO-CREATE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -315,7 +315,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"REPO-WAIT\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"REPO-WAIT\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -324,7 +324,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"REPO-WAIT\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"REPO-WAIT\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"REPO-WAIT\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"REPO-WAIT\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -333,7 +333,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"REPO-WAIT\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"REPO-WAIT\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -392,7 +392,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"REPO-WAIT\"})", + "expr": "sum(porch_perf_operations_total{operation=\"REPO-WAIT\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -400,7 +400,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"REPO-WAIT\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"REPO-WAIT\"})", + "expr": "sum(porch_perf_operations_total{operation=\"REPO-WAIT\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"REPO-WAIT\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -457,7 +457,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"LIST\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"LIST\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -466,7 +466,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"LIST\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"LIST\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"LIST\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"LIST\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -475,7 +475,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"LIST\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"LIST\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -534,7 +534,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"LIST\"})", + "expr": "sum(porch_perf_operations_total{operation=\"LIST\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -542,7 +542,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"LIST\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"LIST\"})", + "expr": "sum(porch_perf_operations_total{operation=\"LIST\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"LIST\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -599,7 +599,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -608,7 +608,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -617,7 +617,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -676,7 +676,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GET\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GET\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -684,7 +684,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GET\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GET\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GET\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"GET\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -741,7 +741,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-PROPOSED\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-PROPOSED\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -750,7 +750,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET-PROPOSED\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET-PROPOSED\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET-PROPOSED\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET-PROPOSED\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -759,7 +759,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-PROPOSED\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-PROPOSED\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -818,7 +818,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GET-PROPOSED\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GET-PROPOSED\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -826,7 +826,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GET-PROPOSED\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GET-PROPOSED\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GET-PROPOSED\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"GET-PROPOSED\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -883,7 +883,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-RESOURCES\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-RESOURCES\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -892,7 +892,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET-RESOURCES\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET-RESOURCES\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"GET-RESOURCES\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"GET-RESOURCES\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -901,7 +901,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-RESOURCES\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"GET-RESOURCES\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -960,7 +960,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GET-RESOURCES\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GET-RESOURCES\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -968,7 +968,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"GET-RESOURCES\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"GET-RESOURCES\"})", + "expr": "sum(porch_perf_operations_total{operation=\"GET-RESOURCES\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"GET-RESOURCES\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1025,7 +1025,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"CREATE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"CREATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -1034,7 +1034,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"CREATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"CREATE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"CREATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"CREATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -1043,7 +1043,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"CREATE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"CREATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -1102,7 +1102,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"CREATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"CREATE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -1110,7 +1110,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"CREATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"CREATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"CREATE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"CREATE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1167,7 +1167,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"UPDATE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"UPDATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -1176,7 +1176,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"UPDATE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"UPDATE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"UPDATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"UPDATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -1185,7 +1185,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"UPDATE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"UPDATE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -1244,7 +1244,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"UPDATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"UPDATE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -1252,7 +1252,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"UPDATE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"UPDATE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"UPDATE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"UPDATE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1309,7 +1309,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -1318,7 +1318,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PROPOSE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PROPOSE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PROPOSE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PROPOSE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -1327,7 +1327,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -1386,7 +1386,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -1394,7 +1394,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"PROPOSE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"PROPOSE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1451,7 +1451,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"APPROVE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"APPROVE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -1460,7 +1460,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"APPROVE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"APPROVE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"APPROVE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"APPROVE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -1469,7 +1469,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"APPROVE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"APPROVE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -1528,7 +1528,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"APPROVE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"APPROVE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -1536,7 +1536,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"APPROVE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"APPROVE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"APPROVE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"APPROVE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1593,7 +1593,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE-DELETION\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE-DELETION\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -1602,7 +1602,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PROPOSE-DELETION\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PROPOSE-DELETION\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"PROPOSE-DELETION\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"PROPOSE-DELETION\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -1611,7 +1611,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE-DELETION\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"PROPOSE-DELETION\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -1670,7 +1670,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\"})", + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -1678,7 +1678,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\"})", + "expr": "sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"PROPOSE-DELETION\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1735,7 +1735,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"DELETE\"}[5m])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"DELETE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p95 duration", "refId": "A" }, @@ -1744,7 +1744,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"DELETE\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"DELETE\"}[5m]))", + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"DELETE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"DELETE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg duration", "refId": "B" }, @@ -1753,7 +1753,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"DELETE\"}[5m])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"DELETE\",api_version=\"$api_version\"}[5m])))", "legendFormat": "p99 duration", "refId": "C" } @@ -1812,7 +1812,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"DELETE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"DELETE\",api_version=\"$api_version\"})", "refId": "A" }, { @@ -1820,7 +1820,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{operation=\"DELETE\",status=\"success\"}) / sum(porch_perf_operations_total{operation=\"DELETE\"})", + "expr": "sum(porch_perf_operations_total{operation=\"DELETE\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"DELETE\",api_version=\"$api_version\"})", "refId": "B" } ], @@ -1869,7 +1869,7 @@ "h": 8, "w": 24, "x": 0, - "y": 90 + "y": 136 }, "targets": [ { @@ -1877,7 +1877,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total{status=\"success\"}) by (operation) / sum(porch_perf_operations_total) by (operation)", + "expr": "sum(porch_perf_operations_total{status=\"success\",api_version=\"$api_version\"}) by (operation) / sum(porch_perf_operations_total{api_version=\"$api_version\"}) by (operation)", "legendFormat": "{{operation}}", "refId": "A" } @@ -1930,7 +1930,7 @@ "h": 8, "w": 12, "x": 0, - "y": 98 + "y": 144 }, "targets": [ { @@ -1938,7 +1938,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum by (operation) (rate(porch_perf_operation_duration_seconds_sum[5m])) / sum by (operation) (rate(porch_perf_operation_duration_seconds_count[5m]))", + "expr": "sum by (operation) (rate(porch_perf_operation_duration_seconds_sum{api_version=\"$api_version\"}[5m])) / sum by (operation) (rate(porch_perf_operation_duration_seconds_count{api_version=\"$api_version\"}[5m]))", "legendFormat": "{{operation}}", "refId": "A" } @@ -1989,7 +1989,7 @@ "h": 8, "w": 12, "x": 12, - "y": 98 + "y": 144 }, "targets": [ { @@ -1997,7 +1997,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_operations_total) by (operation)", + "expr": "sum(porch_perf_operations_total{api_version=\"$api_version\"}) by (operation)", "legendFormat": "{{operation}}", "refId": "A" } @@ -2048,7 +2048,7 @@ "h": 8, "w": 12, "x": 0, - "y": 106 + "y": 152 }, "targets": [ { @@ -2056,7 +2056,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Draft\", to_state=\"Proposed\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Draft\", to_state=\"Proposed\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "Draft -> Proposed (p99)", "refId": "A" }, @@ -2065,7 +2065,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Draft\", to_state=\"Proposed\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Draft\", to_state=\"Proposed\"}[5m]))", + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Draft\", to_state=\"Proposed\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Draft\", to_state=\"Proposed\",api_version=\"$api_version\"}[5m]))", "legendFormat": "Draft -> Proposed (avg)", "refId": "B" }, @@ -2074,7 +2074,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Proposed\", to_state=\"Published\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Proposed\", to_state=\"Published\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "Proposed -> Published (p99)", "refId": "D" }, @@ -2083,7 +2083,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Proposed\", to_state=\"Published\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Proposed\", to_state=\"Published\"}[5m]))", + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Proposed\", to_state=\"Published\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Proposed\", to_state=\"Published\",api_version=\"$api_version\"}[5m]))", "legendFormat": "Proposed -> Published (avg)", "refId": "E" }, @@ -2092,7 +2092,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Published\", to_state=\"DeletionProposed\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Published\", to_state=\"DeletionProposed\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "Published -> DeletionProposed (p99)", "refId": "F" }, @@ -2101,7 +2101,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Published\", to_state=\"DeletionProposed\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Published\", to_state=\"DeletionProposed\"}[5m]))", + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Published\", to_state=\"DeletionProposed\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Published\", to_state=\"DeletionProposed\",api_version=\"$api_version\"}[5m]))", "legendFormat": "Published -> DeletionProposed (avg)", "refId": "G" }, @@ -2110,7 +2110,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Proposed\", to_state=\"DeletionProposed\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"Proposed\", to_state=\"DeletionProposed\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "Proposed -> DeletionProposed (p99)", "refId": "H" }, @@ -2119,7 +2119,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Proposed\", to_state=\"DeletionProposed\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Proposed\", to_state=\"DeletionProposed\"}[5m]))", + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"Proposed\", to_state=\"DeletionProposed\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"Proposed\", to_state=\"DeletionProposed\",api_version=\"$api_version\"}[5m]))", "legendFormat": "Proposed -> DeletionProposed (avg)", "refId": "I" }, @@ -2128,7 +2128,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"DeletionProposed\", to_state=\"deleted\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(porch_perf_lifecycle_transition_duration_seconds_bucket{from_state=\"DeletionProposed\", to_state=\"deleted\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "DeletionProposed -> deleted (p99)", "refId": "J" }, @@ -2137,7 +2137,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"DeletionProposed\", to_state=\"deleted\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"DeletionProposed\", to_state=\"deleted\"}[5m]))", + "expr": "sum(rate(porch_perf_lifecycle_transition_duration_seconds_sum{from_state=\"DeletionProposed\", to_state=\"deleted\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_lifecycle_transition_duration_seconds_count{from_state=\"DeletionProposed\", to_state=\"deleted\",api_version=\"$api_version\"}[5m]))", "legendFormat": "DeletionProposed -> deleted (avg)", "refId": "K" } @@ -2188,7 +2188,7 @@ "h": 8, "w": 12, "x": 12, - "y": 106 + "y": 152 }, "targets": [ { @@ -2196,7 +2196,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(porch_perf_active_operations) by (operation)", + "expr": "sum(porch_perf_active_operations{api_version=\"$api_version\"}) by (operation)", "legendFormat": "{{operation}}", "refId": "A" } @@ -2247,7 +2247,7 @@ "h": 4, "w": 8, "x": 0, - "y": 114 + "y": 160 }, "targets": [ { @@ -2277,7 +2277,7 @@ "h": 4, "w": 8, "x": 8, - "y": 114 + "y": 160 }, "targets": [ { @@ -2307,7 +2307,7 @@ "h": 4, "w": 8, "x": 16, - "y": 114 + "y": 160 }, "targets": [ { @@ -2324,6 +2324,432 @@ "graphMode": "area", "textMode": "value_and_name" } + }, + { + "id": 39, + "title": "WAIT-READY - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 118 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"WAIT-READY\",api_version=\"$api_version\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"WAIT-READY\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"WAIT-READY\",api_version=\"$api_version\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"WAIT-READY\",api_version=\"$api_version\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 40, + "title": "WAIT-READY - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 118 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"WAIT-READY\",api_version=\"$api_version\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"WAIT-READY\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"WAIT-READY\",api_version=\"$api_version\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 41, + "title": "WAIT-RENDERED - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 124 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"WAIT-RENDERED\",api_version=\"$api_version\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"WAIT-RENDERED\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"WAIT-RENDERED\",api_version=\"$api_version\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"WAIT-RENDERED\",api_version=\"$api_version\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 42, + "title": "WAIT-RENDERED - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 124 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"WAIT-RENDERED\",api_version=\"$api_version\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"WAIT-RENDERED\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"WAIT-RENDERED\",api_version=\"$api_version\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } + }, + { + "id": 43, + "title": "WAIT-PUBLISHED - Duration & Success Rate", + "type": "graph", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 130 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"WAIT-PUBLISHED\",api_version=\"$api_version\"}[5m])))", + "legendFormat": "p95 duration", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_perf_operation_duration_seconds_sum{operation=\"WAIT-PUBLISHED\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_perf_operation_duration_seconds_count{operation=\"WAIT-PUBLISHED\",api_version=\"$api_version\"}[5m]))", + "legendFormat": "avg duration", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(porch_perf_operation_duration_seconds_bucket{operation=\"WAIT-PUBLISHED\",api_version=\"$api_version\"}[5m])))", + "legendFormat": "p99 duration", + "refId": "C" + } + ], + "yaxes": [ + { + "format": "s", + "label": "Duration", + "show": true + }, + { + "format": "short", + "show": true + } + ], + "xaxis": { + "show": true, + "mode": "time", + "name": null, + "values": [] + }, + "grid": { + "show": true, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "lines": true, + "linewidth": 1, + "pointradius": 5, + "points": false + }, + { + "id": 44, + "title": "WAIT-PUBLISHED - Total & Success Rate", + "type": "stat", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 130 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"WAIT-PUBLISHED\",api_version=\"$api_version\"})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(porch_perf_operations_total{operation=\"WAIT-PUBLISHED\",status=\"success\",api_version=\"$api_version\"}) / sum(porch_perf_operations_total{operation=\"WAIT-PUBLISHED\",api_version=\"$api_version\"})", + "refId": "B" + } + ], + "options": { + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "displayName", + "value": "Success Rate" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + } } ], "templating": { @@ -2337,6 +2763,16 @@ "value": "porch-metrics" }, "refresh": 1 + }, + { + "name": "api_version", + "type": "query", + "query": "label_values(porch_perf_test_run_info, api_version)", + "current": { + "text": "v1alpha1", + "value": "v1alpha1" + }, + "refresh": 1 } ] }, @@ -2355,4 +2791,4 @@ "1d" ] } -} \ No newline at end of file +} diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index ed181c38f..17c6a32e5 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -225,9 +225,10 @@ func RecordPackageRevisionResourcesSize(ctx context.Context, prKey repository.Pa } // Performance test metric recording functions -func PerfTestRecordMetric(operation, repoName, pkgName string, duration time.Duration, err error) { +func PerfTestRecordMetric(operation, apiVersion, repoName, pkgName string, duration time.Duration, err error) { attrs := metric.WithAttributes( attribute.String("operation", operation), + attribute.String("api_version", apiVersion), attribute.String("repository", repoName), attribute.String("package", pkgName), attribute.String("status", statusLabel(err)), @@ -237,11 +238,12 @@ func PerfTestRecordMetric(operation, repoName, pkgName string, duration time.Dur perfOperationCounter.Add(ctx, 1, attrs) } -func PerfTestRecordLifecycleTransition(fromState, toState, repoName, pkgName string, duration time.Duration, err error) { +func PerfTestRecordLifecycleTransition(fromState, toState, apiVersion, repoName, pkgName string, duration time.Duration, err error) { perfLifecycleTransitionDuration.Record(context.Background(), duration.Seconds(), metric.WithAttributes( attribute.String("from_state", fromState), attribute.String("to_state", toState), + attribute.String("api_version", apiVersion), attribute.String("repository", repoName), attribute.String("package", pkgName), attribute.String("status", statusLabel(err)), @@ -258,19 +260,23 @@ func PerfTestRecordPackageRevision(operation string, err error) { ) } -func PerfTestSetTestRunInfo(testName, namespace string, startTime time.Time) { +func PerfTestSetTestRunInfo(testName, namespace, apiVersion string, startTime time.Time) { perfTestRunInfoGauge.Record(context.Background(), 1, metric.WithAttributes( attribute.String("test_name", testName), attribute.String("namespace", namespace), + attribute.String("api_version", apiVersion), attribute.String("start_time", startTime.Format(time.RFC3339)), ), ) } -func PerfTestRecordActiveOperation(operation string, delta float64) { +func PerfTestRecordActiveOperation(operation, apiVersion string, delta float64) { perfActiveOperations.Add(context.Background(), delta, - metric.WithAttributes(attribute.String("operation", operation)), + metric.WithAttributes( + attribute.String("operation", operation), + attribute.String("api_version", apiVersion), + ), ) } diff --git a/test/performance/csv_generation.go b/test/performance/csv_generation.go index 2aebee43d..2d8815586 100644 --- a/test/performance/csv_generation.go +++ b/test/performance/csv_generation.go @@ -68,12 +68,10 @@ func (t *PerfTestSuite) generateCSVResults() error { if lifecycleComplete { for opKey, opMetric := range revMetrics.Metrics { - if opMetric.Error != nil { + if opMetric.Error != nil || isDeletionOperation(opKey) { continue } - if opKey != pkgRevProposeDeletion && opKey != pkgRevDelete { - totalLifecycleDur += opMetric.Duration - } + totalLifecycleDur += opMetric.Duration } } @@ -136,20 +134,15 @@ func (t *PerfTestSuite) generateDetailedOperationsCSV() error { writer := csv.NewWriter(csvFile) defer writer.Flush() + lifecycleOps := t.testOptions.apiVersion.LifecyclePkgRevOperations() + header := []string{} for i := 0; i < t.testOptions.numRepos; i++ { repoSuffix := fmt.Sprintf("%0*d", len(fmt.Sprintf("%d", t.testOptions.numRepos)), i) - header = append(header, - fmt.Sprintf("REPO-%s-PKG:REV", repoSuffix), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevList), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevCreate), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevResourcesGet), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevUpdate), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevGet), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevPropose), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevGetProposed), - fmt.Sprintf("REPO-%s-%s", repoSuffix, pkgRevPublished), - ) + header = append(header, fmt.Sprintf("REPO-%s-PKG:REV", repoSuffix)) + for _, op := range lifecycleOps { + header = append(header, fmt.Sprintf("REPO-%s-%s", repoSuffix, op)) + } } if err := writer.Write(header); err != nil { @@ -171,13 +164,10 @@ func (t *PerfTestSuite) generateDetailedOperationsCSV() error { ops := make(map[string]time.Duration) for opKey, opMetric := range revMetrics.Metrics { - if opMetric.Error != nil { + if opMetric.Error != nil || isDeletionOperation(opKey) { continue } - - if opKey != pkgRevProposeDeletion && opKey != pkgRevDelete { - ops[opKey] = opMetric.Duration - } + ops[opKey] = opMetric.Duration } repoOps[repoName] = append(repoOps[repoName], pkgRevOps{ @@ -205,6 +195,11 @@ func (t *PerfTestSuite) generateDetailedOperationsCSV() error { } } + emptyOpValues := make([]string, len(lifecycleOps)) + for i := range emptyOpValues { + emptyOpValues[i] = "0" + } + for row := 0; row < maxRows; row++ { record := []string{} for i := 0; i < t.testOptions.numRepos; i++ { @@ -216,18 +211,12 @@ func (t *PerfTestSuite) generateDetailedOperationsCSV() error { pkgRev := fmt.Sprintf("%s:v%d", op.pkgName, op.revision) record = append(record, pkgRev) - record = append(record, - formatDuration(op.ops[pkgRevList]), - formatDuration(op.ops[pkgRevCreate]), - formatDuration(op.ops[pkgRevResourcesGet]), - formatDuration(op.ops[pkgRevUpdate]), - formatDuration(op.ops[pkgRevGet]), - formatDuration(op.ops[pkgRevPropose]), - formatDuration(op.ops[pkgRevGetProposed]), - formatDuration(op.ops[pkgRevPublished]), - ) + for _, opKey := range lifecycleOps { + record = append(record, formatDuration(op.ops[opKey])) + } } else { - record = append(record, "", "0", "0", "0", "0", "0", "0", "0", "0") + record = append(record, "") + record = append(record, emptyOpValues...) } } diff --git a/test/performance/driver.go b/test/performance/driver.go new file mode 100644 index 000000000..df658d340 --- /dev/null +++ b/test/performance/driver.go @@ -0,0 +1,349 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + "strings" + "time" + + porchapi "github.com/kptdev/porch/api/porch/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// LifecycleDriver implements package revision lifecycle operations for a specific Porch API version. +type LifecycleDriver interface { + Version() PorchAPIVersion + RepositoryAnnotations() map[string]string + DoLifecycle(repoName, pkgName string, revisionNum int) (pkgRevName string, err error) + DeletePackageRevision(repoName, pkgName, pkgRevName string, revisionNum int) error + ListPackageRevisionsForDeletion() ([]DeletionCandidate, error) +} + +func NewLifecycleDriver(version PorchAPIVersion, suite *PerfTestSuite) (LifecycleDriver, error) { + switch version { + case PorchAPIV1Alpha1: + return &v1alpha1Driver{baseDriver: baseDriver{suite: suite}}, nil + case PorchAPIV1Alpha2: + return &v1alpha2Driver{baseDriver: baseDriver{suite: suite}}, nil + default: + return nil, fmt.Errorf("unsupported porch API version %q", version) + } +} + +type baseDriver struct { + suite *PerfTestSuite +} + +type deletionCandidateExtractor func(client.Object) (DeletionCandidate, bool) + +func (b *baseDriver) createPackageRevision(obj client.Object, repoName, pkgName string, revisionNum int) error { + t := b.suite + start := time.Now() + if t.enablePrometheus { + apiVersion := string(t.testOptions.apiVersion) + telemetry.PerfTestRecordActiveOperation(pkgRevCreate, apiVersion, 1) + defer telemetry.PerfTestRecordActiveOperation(pkgRevCreate, apiVersion, -1) + } + + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Create(t.ctx, obj) + }) + duration := time.Since(start) + + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevCreate, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevCreate, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + t.recordPerfMetric(pkgRevCreate, repoName, pkgName, duration, err) + if t.enablePrometheus { + telemetry.PerfTestRecordPackageRevision(pkgRevCreate, err) + } + + return err +} + +func (b *baseDriver) updatePackageRevisionResources(repoName, pkgName, pkgRevName string, revisionNum int) error { + t := b.suite + var resources porchapi.PackageRevisionResources + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &resources) + }) + duration := time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevResourcesGet, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevResourcesGet, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + t.recordPerfMetric(pkgRevResourcesGet, repoName, pkgName, duration, err) + if err != nil { + return err + } + + pkgResources := t.createPackageResources() + if resources.Spec.Resources == nil { + resources.Spec.Resources = make(map[string]string) + } + for name, content := range pkgResources { + resources.Spec.Resources[name] = content + } + + start = time.Now() + err = retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Update(t.ctx, &resources) + }) + duration = time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevUpdate, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevUpdate, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + t.recordPerfMetric(pkgRevUpdate, repoName, pkgName, duration, err) + + return err +} + +type deletionBehavior struct { + alwaysProposeDeletion bool + publishedLifecycle string + deletionProposed string + afterProposeDeletion func(repoName, pkgName, pkgRevName string, revisionNum int) error + waitForDeleted func(pkgRevName string) error +} + +func (b *baseDriver) deletePackageRevision( + repoName, pkgName, pkgRevName string, + revisionNum int, + newObj func() client.Object, + getLifecycle func(client.Object) string, + proposeDeletion func(client.Object) error, + behavior deletionBehavior, +) error { + t := b.suite + obj := newObj() + + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj) + }) + if err != nil { + return err + } + + initialLifecycle := getLifecycle(obj) + shouldPropose := behavior.alwaysProposeDeletion || initialLifecycle == behavior.publishedLifecycle + + start := time.Now() + if shouldPropose { + err = retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj); err != nil { + return err + } + return proposeDeletion(obj) + }) + } + duration := time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevProposeDeletion, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevProposeDeletion, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + t.recordPerfMetric(pkgRevProposeDeletion, repoName, pkgName, duration, err) + if t.enablePrometheus && shouldPropose { + telemetry.PerfTestRecordLifecycleTransition(initialLifecycle, behavior.deletionProposed, string(t.testOptions.apiVersion), repoName, pkgName, duration, err) + } + if err != nil { + return err + } + + if shouldPropose && behavior.afterProposeDeletion != nil { + if err = behavior.afterProposeDeletion(repoName, pkgName, pkgRevName, revisionNum); err != nil { + return err + } + } + + start = time.Now() + err = retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj); err != nil { + return err + } + return t.client.Delete(t.ctx, obj) + }) + if err == nil && behavior.waitForDeleted != nil { + if waitErr := behavior.waitForDeleted(pkgRevName); waitErr != nil { + err = waitErr + } + } + duration = time.Since(start) + t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevDelete, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", pkgRevDelete, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + t.recordPerfMetric(pkgRevDelete, repoName, pkgName, duration, err) + if t.enablePrometheus && shouldPropose { + telemetry.PerfTestRecordLifecycleTransition(behavior.deletionProposed, "deleted", string(t.testOptions.apiVersion), repoName, pkgName, duration, err) + } + + return err +} + +func (b *baseDriver) listPackageRevisionsForDeletion(list client.ObjectList, extract deletionCandidateExtractor) ([]DeletionCandidate, error) { + t := b.suite + if err := t.client.List(t.ctx, list, client.InNamespace(t.testOptions.namespace)); err != nil { + return nil, err + } + + items, err := meta.ExtractList(list) + if err != nil { + return nil, err + } + + prefix := fmt.Sprintf("%s-test-", t.testOptions.namespace) + candidates := make([]DeletionCandidate, 0, len(items)) + for _, item := range items { + obj, ok := item.(client.Object) + if !ok { + continue + } + candidate, ok := extract(obj) + if !ok || !strings.HasPrefix(candidate.RepoName, prefix) { + continue + } + candidates = append(candidates, candidate) + } + + return candidates, nil +} + +func (b *baseDriver) recordListOperation(repoName, pkgName string, revisionNum int, start time.Time, err error) { + b.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevList, start, err) +} + +func (b *baseDriver) recordPkgRevOperation(repoName, pkgName string, revisionNum int, opKey string, start time.Time, err error) { + duration := time.Since(start) + b.suite.recordPkgRevMetric(repoName, pkgName, revisionNum, opKey, OperationMetrics{ + Operation: fmt.Sprintf("%s:%d", opKey, revisionNum), + Duration: duration, + Error: err, + Timestamp: start, + }) + b.suite.recordPerfMetric(opKey, repoName, pkgName, duration, err) +} + +func (b *baseDriver) getPackageRevision(pkgRevName, repoName, pkgName string, revisionNum int, opKey string, obj client.Object) error { + t := b.suite + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj) + }) + b.recordPkgRevOperation(repoName, pkgName, revisionNum, opKey, start, err) + return err +} + +func (b *baseDriver) proposePackage( + repoName, pkgName, pkgRevName string, + revisionNum int, + obj client.Object, + getLifecycle func(client.Object) string, + setProposed func(client.Object) error, + proposedLifecycle string, +) error { + t := b.suite + if err := b.getPackageRevision(pkgRevName, repoName, pkgName, revisionNum, pkgRevGet, obj); err != nil { + return err + } + + start := time.Now() + initialLifecycle := getLifecycle(obj) + err := retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj); err != nil { + return err + } + return setProposed(obj) + }) + b.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevPropose, start, err) + if t.enablePrometheus { + telemetry.PerfTestRecordLifecycleTransition(initialLifecycle, proposedLifecycle, string(t.testOptions.apiVersion), repoName, pkgName, time.Since(start), err) + } + + return err +} + +func (b *baseDriver) approvePackage( + repoName, pkgName, pkgRevName string, + revisionNum int, + obj client.Object, + publish func(client.Object) error, + proposedLifecycle, publishedLifecycle string, +) error { + t := b.suite + if err := b.getPackageRevision(pkgRevName, repoName, pkgName, revisionNum, pkgRevGetProposed, obj); err != nil { + return err + } + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj); err != nil { + return err + } + return publish(obj) + }) + b.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevPublished, start, err) + if t.enablePrometheus { + telemetry.PerfTestRecordLifecycleTransition(proposedLifecycle, publishedLifecycle, string(t.testOptions.apiVersion), repoName, pkgName, time.Since(start), err) + } + + return err +} + +func (t *PerfTestSuite) recordPerfMetric(operation, repoName, pkgName string, duration time.Duration, err error) { + if !t.enablePrometheus { + return + } + telemetry.PerfTestRecordMetric(operation, string(t.testOptions.apiVersion), repoName, pkgName, duration, err) +} + +func packageRevisionCRDName(repo, pkg, workspace string) string { + return fmt.Sprintf("%s.%s.%s", repo, pkg, workspace) +} + +func workspaceRevisionNum(workspace string) int { + revisionNum := 1 + if strings.Contains(workspace, "v") { + _, _ = fmt.Sscanf(workspace, "v%d", &revisionNum) + } + return revisionNum +} + +func deletionCandidateFromFields(name, repoName, packageName, workspaceName string) DeletionCandidate { + return DeletionCandidate{ + Name: name, + RepoName: repoName, + PackageName: packageName, + WorkspaceName: workspaceName, + RevisionNum: workspaceRevisionNum(workspaceName), + } +} diff --git a/test/performance/driver_v1alpha1.go b/test/performance/driver_v1alpha1.go new file mode 100644 index 000000000..66e283565 --- /dev/null +++ b/test/performance/driver_v1alpha1.go @@ -0,0 +1,180 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + "time" + + porchv1alpha1 "github.com/kptdev/porch/api/porch/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type v1alpha1Driver struct { + baseDriver +} + +func (d *v1alpha1Driver) Version() PorchAPIVersion { + return PorchAPIV1Alpha1 +} + +func (d *v1alpha1Driver) RepositoryAnnotations() map[string]string { + return nil +} + +func (d *v1alpha1Driver) DoLifecycle(repoName, pkgName string, revisionNum int) (string, error) { + t := d.suite + var list porchv1alpha1.PackageRevisionList + var taskList []porchv1alpha1.Task + + t.initPkgRevMetrics(repoName, pkgName, revisionNum) + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.List(t.ctx, &list, client.InNamespace(t.testOptions.namespace)) + }) + d.recordListOperation(repoName, pkgName, revisionNum, start, err) + if err != nil { + return "", err + } + + var latestPR *porchv1alpha1.PackageRevision + for i := range list.Items { + pr := &list.Items[i] + if pr.Spec.PackageName == pkgName && + pr.Spec.RepositoryName == repoName && + pr.Spec.Lifecycle == porchv1alpha1.PackageRevisionLifecyclePublished { + if latestPR == nil || pr.Spec.Revision > latestPR.Spec.Revision { + latestPR = pr + } + } + } + + if revisionNum == 1 { + taskList = []porchv1alpha1.Task{ + { + Type: porchv1alpha1.TaskTypeInit, + Init: &porchv1alpha1.PackageInitTaskSpec{ + Description: fmt.Sprintf("Test package %s for Porch metrics", pkgName), + Keywords: []string{"test", "metrics"}, + Site: "https://kpt.dev/", + }, + }, + } + if t.enablePrometheus { + telemetry.PerfTestIncrementPackageCounter() + } + } else if latestPR != nil { + taskList = []porchv1alpha1.Task{ + { + Type: porchv1alpha1.TaskTypeEdit, + Edit: &porchv1alpha1.PackageEditTaskSpec{ + Source: &porchv1alpha1.PackageRevisionRef{ + Name: latestPR.Name, + }, + }, + }, + } + } + + workspace := fmt.Sprintf("v%d", revisionNum) + pkgRev := &porchv1alpha1.PackageRevision{ + TypeMeta: metav1.TypeMeta{ + Kind: "PackageRevision", + APIVersion: porchv1alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.testOptions.namespace, + }, + Spec: porchv1alpha1.PackageRevisionSpec{ + PackageName: pkgName, + WorkspaceName: workspace, + RepositoryName: repoName, + Tasks: taskList, + }, + } + + if err = d.createPackageRevision(pkgRev, repoName, pkgName, revisionNum); err != nil { + return "", err + } + + if err = d.updatePackageRevisionResources(repoName, pkgName, pkgRev.Name, revisionNum); err != nil { + return "", err + } + + pkg := &porchv1alpha1.PackageRevision{} + if err = d.proposePackage( + repoName, pkgName, pkgRev.Name, revisionNum, + pkg, + func(obj client.Object) string { return string(obj.(*porchv1alpha1.PackageRevision).Spec.Lifecycle) }, + func(obj client.Object) error { + pr := obj.(*porchv1alpha1.PackageRevision) + pr.Spec.Lifecycle = porchv1alpha1.PackageRevisionLifecycleProposed + return t.client.Update(t.ctx, pr) + }, + string(porchv1alpha1.PackageRevisionLifecycleProposed), + ); err != nil { + return "", err + } + + if err = d.approvePackage( + repoName, pkgName, pkgRev.Name, revisionNum, + pkg, + func(obj client.Object) error { + pr := obj.(*porchv1alpha1.PackageRevision) + pr.Spec.Lifecycle = porchv1alpha1.PackageRevisionLifecyclePublished + _, err := t.clientSet.PorchV1alpha1().PackageRevisions(t.testOptions.namespace).UpdateApproval(t.ctx, pkgRev.Name, pr, metav1.UpdateOptions{}) + return err + }, + string(porchv1alpha1.PackageRevisionLifecycleProposed), + string(porchv1alpha1.PackageRevisionLifecyclePublished), + ); err != nil { + return "", err + } + + return pkgRev.Name, nil +} + +func (d *v1alpha1Driver) DeletePackageRevision(repoName, pkgName, pkgRevName string, revisionNum int) error { + return d.deletePackageRevision( + repoName, pkgName, pkgRevName, revisionNum, + func() client.Object { return &porchv1alpha1.PackageRevision{} }, + func(obj client.Object) string { + return string(obj.(*porchv1alpha1.PackageRevision).Spec.Lifecycle) + }, + func(obj client.Object) error { + pkgRev := obj.(*porchv1alpha1.PackageRevision) + pkgRev.Spec.Lifecycle = porchv1alpha1.PackageRevisionLifecycleDeletionProposed + return d.suite.client.Update(d.suite.ctx, pkgRev) + }, + deletionBehavior{ + alwaysProposeDeletion: true, + deletionProposed: string(porchv1alpha1.PackageRevisionLifecycleDeletionProposed), + }, + ) +} + +func (d *v1alpha1Driver) ListPackageRevisionsForDeletion() ([]DeletionCandidate, error) { + return d.listPackageRevisionsForDeletion(&porchv1alpha1.PackageRevisionList{}, func(obj client.Object) (DeletionCandidate, bool) { + pr, ok := obj.(*porchv1alpha1.PackageRevision) + if !ok { + return DeletionCandidate{}, false + } + return deletionCandidateFromFields(pr.Name, pr.Spec.RepositoryName, pr.Spec.PackageName, pr.Spec.WorkspaceName), true + }) +} diff --git a/test/performance/driver_v1alpha2.go b/test/performance/driver_v1alpha2.go new file mode 100644 index 000000000..1da168ea7 --- /dev/null +++ b/test/performance/driver_v1alpha2.go @@ -0,0 +1,322 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + "time" + + porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" + "github.com/kptdev/porch/internal/telemetry" + pkgerrors "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const v1alpha2ControllerWaitTimeout = 120 * time.Second + +type v1alpha2Driver struct { + baseDriver +} + +func (d *v1alpha2Driver) Version() PorchAPIVersion { + return PorchAPIV1Alpha2 +} + +func (d *v1alpha2Driver) RepositoryAnnotations() map[string]string { + return map[string]string{ + "porch.kpt.dev/v1alpha2-migration": "true", + } +} + +func (d *v1alpha2Driver) DoLifecycle(repoName, pkgName string, revisionNum int) (string, error) { + t := d.suite + var list porchv1alpha2.PackageRevisionList + + t.initPkgRevMetrics(repoName, pkgName, revisionNum) + + start := time.Now() + err := retry.RetryOnConflict(retryBackoff, func() error { + return t.client.List(t.ctx, &list, client.InNamespace(t.testOptions.namespace)) + }) + d.recordListOperation(repoName, pkgName, revisionNum, start, err) + if err != nil { + return "", err + } + + var latestPR *porchv1alpha2.PackageRevision + for i := range list.Items { + pr := &list.Items[i] + if pr.Spec.PackageName == pkgName && + pr.Spec.RepositoryName == repoName && + pr.Spec.Lifecycle == porchv1alpha2.PackageRevisionLifecyclePublished { + if latestPR == nil || pr.Status.Revision > latestPR.Status.Revision { + latestPR = pr + } + } + } + + workspace := fmt.Sprintf("v%d", revisionNum) + pkgRev := &porchv1alpha2.PackageRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: packageRevisionCRDName(repoName, pkgName, workspace), + Namespace: t.testOptions.namespace, + }, + Spec: porchv1alpha2.PackageRevisionSpec{ + PackageName: pkgName, + RepositoryName: repoName, + WorkspaceName: workspace, + Lifecycle: porchv1alpha2.PackageRevisionLifecycleDraft, + }, + } + + if revisionNum == 1 { + pkgRev.Spec.Source = &porchv1alpha2.PackageSource{ + Init: &porchv1alpha2.PackageInitSpec{ + Description: fmt.Sprintf("Test package %s for Porch metrics", pkgName), + Keywords: []string{"test", "metrics"}, + Site: "https://kpt.dev/", + }, + } + if t.enablePrometheus { + telemetry.PerfTestIncrementPackageCounter() + } + } else if latestPR != nil { + pkgRev.Spec.Source = &porchv1alpha2.PackageSource{ + CopyFrom: &porchv1alpha2.PackageRevisionRef{ + Name: latestPR.Name, + }, + } + } + + if err = d.createPackageRevision(pkgRev, repoName, pkgName, revisionNum); err != nil { + return "", err + } + + if err = d.waitForReady(pkgRev, repoName, pkgName, revisionNum); err != nil { + return "", err + } + + if err = d.updatePackageRevisionResources(repoName, pkgName, pkgRev.Name, revisionNum); err != nil { + return "", err + } + + if err = d.waitForRendered(pkgRev, repoName, pkgName, revisionNum); err != nil { + return "", err + } + + if err = d.proposePackage( + repoName, pkgName, pkgRev.Name, revisionNum, + pkgRev, + func(obj client.Object) string { return string(obj.(*porchv1alpha2.PackageRevision).Spec.Lifecycle) }, + func(obj client.Object) error { + pr := obj.(*porchv1alpha2.PackageRevision) + pr.Spec.Lifecycle = porchv1alpha2.PackageRevisionLifecycleProposed + return t.client.Update(t.ctx, pr) + }, + string(porchv1alpha2.PackageRevisionLifecycleProposed), + ); err != nil { + return "", err + } + + if err = d.waitForReady(pkgRev, repoName, pkgName, revisionNum); err != nil { + return "", err + } + + if err = d.approvePackage( + repoName, pkgName, pkgRev.Name, revisionNum, + pkgRev, + func(obj client.Object) error { + pr := obj.(*porchv1alpha2.PackageRevision) + pr.Spec.Lifecycle = porchv1alpha2.PackageRevisionLifecyclePublished + return t.client.Update(t.ctx, pr) + }, + string(porchv1alpha2.PackageRevisionLifecycleProposed), + string(porchv1alpha2.PackageRevisionLifecyclePublished), + ); err != nil { + return "", err + } + + if err = d.waitForPublished(pkgRev, repoName, pkgName, revisionNum); err != nil { + return "", err + } + + return pkgRev.Name, nil +} + +func (d *v1alpha2Driver) waitForReady(pkgRev *porchv1alpha2.PackageRevision, repoName, pkgName string, revisionNum int) error { + start := time.Now() + err := d.pollUntil(v1alpha2ControllerWaitTimeout, func() (bool, error) { + if err := d.suite.ctx.Err(); err != nil { + return false, err + } + if err := d.suite.client.Get(d.suite.ctx, client.ObjectKeyFromObject(pkgRev), pkgRev); err != nil { + return false, err + } + if pkgRev.Status.RenderingPrrResourceVersion != "" { + return false, nil + } + for _, cond := range pkgRev.Status.Conditions { + if cond.Type == porchv1alpha2.ConditionReady && + cond.Status == metav1.ConditionTrue && + cond.ObservedGeneration == pkgRev.Generation { + return true, nil + } + } + return false, nil + }) + d.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevWaitReady, start, err) + return err +} + +func (d *v1alpha2Driver) waitForRendered(pkgRev *porchv1alpha2.PackageRevision, repoName, pkgName string, revisionNum int) error { + start := time.Now() + err := d.pollUntil(v1alpha2ControllerWaitTimeout, func() (bool, error) { + if err := d.suite.ctx.Err(); err != nil { + return false, err + } + if err := d.suite.client.Get(d.suite.ctx, client.ObjectKeyFromObject(pkgRev), pkgRev); err != nil { + return false, err + } + renderReq := pkgRev.Annotations[porchv1alpha2.AnnotationRenderRequest] + if renderReq != "" && pkgRev.Status.ObservedPrrResourceVersion != renderReq { + return false, nil + } + for _, cond := range pkgRev.Status.Conditions { + if cond.Type == porchv1alpha2.ConditionRendered && cond.Status == metav1.ConditionTrue { + return true, nil + } + } + return false, nil + }) + d.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevWaitRendered, start, err) + return err +} + +func (d *v1alpha2Driver) waitForPublished(pkgRev *porchv1alpha2.PackageRevision, repoName, pkgName string, revisionNum int) error { + start := time.Now() + err := d.pollUntil(v1alpha2ControllerWaitTimeout, func() (bool, error) { + if err := d.suite.ctx.Err(); err != nil { + return false, err + } + if err := d.suite.client.Get(d.suite.ctx, client.ObjectKeyFromObject(pkgRev), pkgRev); err != nil { + return false, err + } + return pkgRev.Spec.Lifecycle == porchv1alpha2.PackageRevisionLifecyclePublished && pkgRev.Status.Revision != 0, nil + }) + d.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevWaitPublished, start, err) + return err +} + +func (d *v1alpha2Driver) pollUntil(timeout time.Duration, check func() (bool, error)) error { + deadline := time.Now().Add(timeout) + for { + done, err := check() + if err != nil { + return err + } + if done { + return nil + } + if time.Now().After(deadline) { + return pkgerrors.Errorf("timeout after %v waiting for controller reconciliation", timeout) + } + select { + case <-d.suite.ctx.Done(): + return d.suite.ctx.Err() + case <-time.After(50 * time.Millisecond): + } + } +} + +func (d *v1alpha2Driver) waitUntilReady(pkgRev *porchv1alpha2.PackageRevision) error { + t := d.suite + return d.pollUntil(v1alpha2ControllerWaitTimeout, func() (bool, error) { + if err := t.ctx.Err(); err != nil { + return false, err + } + if err := t.client.Get(t.ctx, client.ObjectKeyFromObject(pkgRev), pkgRev); err != nil { + return false, err + } + if pkgRev.Status.RenderingPrrResourceVersion != "" { + return false, nil + } + for _, cond := range pkgRev.Status.Conditions { + if cond.Type == porchv1alpha2.ConditionReady && + cond.Status == metav1.ConditionTrue && + cond.ObservedGeneration == pkgRev.Generation { + return true, nil + } + } + return false, nil + }) +} + +func (d *v1alpha2Driver) waitForDeleted(pkgRevName string) error { + t := d.suite + return d.pollUntil(v1alpha2ControllerWaitTimeout, func() (bool, error) { + if err := t.ctx.Err(); err != nil { + return false, err + } + obj := &porchv1alpha2.PackageRevision{} + err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, obj) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + }) +} + +func (d *v1alpha2Driver) DeletePackageRevision(repoName, pkgName, pkgRevName string, revisionNum int) error { + return d.deletePackageRevision( + repoName, pkgName, pkgRevName, revisionNum, + func() client.Object { return &porchv1alpha2.PackageRevision{} }, + func(obj client.Object) string { + return string(obj.(*porchv1alpha2.PackageRevision).Spec.Lifecycle) + }, + func(obj client.Object) error { + pkgRev := obj.(*porchv1alpha2.PackageRevision) + pkgRev.Spec.Lifecycle = porchv1alpha2.PackageRevisionLifecycleDeletionProposed + return d.suite.client.Update(d.suite.ctx, pkgRev) + }, + deletionBehavior{ + publishedLifecycle: string(porchv1alpha2.PackageRevisionLifecyclePublished), + deletionProposed: string(porchv1alpha2.PackageRevisionLifecycleDeletionProposed), + afterProposeDeletion: func(repoName, pkgName, pkgRevName string, revisionNum int) error { + pr := &porchv1alpha2.PackageRevision{} + if err := d.suite.client.Get(d.suite.ctx, client.ObjectKey{Namespace: d.suite.testOptions.namespace, Name: pkgRevName}, pr); err != nil { + return err + } + return d.waitUntilReady(pr) + }, + waitForDeleted: d.waitForDeleted, + }, + ) +} + +func (d *v1alpha2Driver) ListPackageRevisionsForDeletion() ([]DeletionCandidate, error) { + return d.listPackageRevisionsForDeletion(&porchv1alpha2.PackageRevisionList{}, func(obj client.Object) (DeletionCandidate, bool) { + pr, ok := obj.(*porchv1alpha2.PackageRevision) + if !ok { + return DeletionCandidate{}, false + } + return deletionCandidateFromFields(pr.Name, pr.Spec.RepositoryName, pr.Spec.PackageName, pr.Spec.WorkspaceName), true + }) +} diff --git a/test/performance/logger.go b/test/performance/logger.go index e84ca13db..bde0d4c22 100644 --- a/test/performance/logger.go +++ b/test/performance/logger.go @@ -34,14 +34,14 @@ type TestLogger struct { mutex sync.Mutex } -func (t *PerfTestSuite) NewTestLogger(prefix string) (*TestLogger, error) { +func (t *PerfTestSuite) NewTestLogger(prefix string, apiVersion PorchAPIVersion) (*TestLogger, error) { logsDir := "logs" if err := os.MkdirAll(logsDir, 0755); err != nil { return nil, pkgerrors.Wrapf(err, "failed to create logs directory %s", logsDir) } timestamp := time.Now().Format(timeDateFormat) - filename := filepath.Join(logsDir, fmt.Sprintf("%s-%s.log", prefix, timestamp)) + filename := filepath.Join(logsDir, fmt.Sprintf("%s-%s-%s.log", prefix, apiVersion, timestamp)) absPath, _ := filepath.Abs(filename) fmt.Printf("Creating test log file: %s\n", absPath) @@ -124,6 +124,22 @@ func (t *PerfTestSuite) NewResultsLogger(resultsFileName, logFileName string) (* }, nil } +func (l *ResultsLogger) LogTestConfig(apiVersion PorchAPIVersion, opts TestOptions, enablePrometheus bool) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.logFileClosed || l.logFile == nil { + return + } + + timestamp := time.Now().Format(timeDateFormat) + _, _ = fmt.Fprintf(l.logFile, + "%s Performance test configuration: api_version=%s namespace=%s repos=%d packages=%d revisions=%d repo_parallelism=%d package_parallelism=%d enable_deletion=%v enable_prometheus=%v\n", + timestamp, apiVersion, opts.namespace, opts.numRepos, opts.numPkgs, opts.numRevs, + opts.repoParallelism, opts.packageParallelism, opts.enableDeletion, enablePrometheus) + _ = l.logFile.Sync() +} + func (l *ResultsLogger) LogApproved(repoName, pkgName string, revision int, prName string, duration time.Duration) { l.mutex.Lock() defer l.mutex.Unlock() diff --git a/test/performance/operations.go b/test/performance/operations.go new file mode 100644 index 000000000..1b73f3565 --- /dev/null +++ b/test/performance/operations.go @@ -0,0 +1,108 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +var repoOperations = []string{ + giteaRepoCreate, + porchRepoCreate, + repoWait, +} + +var v1alpha1PkgRevOperations = []string{ + pkgRevList, + pkgRevCreate, + pkgRevResourcesGet, + pkgRevUpdate, + pkgRevGet, + pkgRevPropose, + pkgRevGetProposed, + pkgRevPublished, + pkgRevProposeDeletion, + pkgRevDelete, +} + +var v1alpha2PkgRevOperations = []string{ + pkgRevList, + pkgRevCreate, + pkgRevWaitReady, + pkgRevResourcesGet, + pkgRevUpdate, + pkgRevWaitRendered, + pkgRevGet, + pkgRevPropose, + pkgRevGetProposed, + pkgRevPublished, + pkgRevWaitPublished, + pkgRevProposeDeletion, + pkgRevDelete, +} + +var operationHeadings = map[string]string{ + giteaRepoCreate: "Create Gitea Repository ", + porchRepoCreate: "Create Porch Repository ", + repoWait: "Repository Ready Wait", + pkgRevList: "Package Revision List", + pkgRevCreate: "Package Revision Create", + pkgRevWaitReady: "Package Revision Wait Ready", + pkgRevResourcesGet: "Package Revision Get Resources", + pkgRevUpdate: "Package Revision Update", + pkgRevWaitRendered: "Package Revision Wait Rendered", + pkgRevGet: "Package Revision Get", + pkgRevPropose: "Package Revision Propose", + pkgRevGetProposed: "Package Revision Get (Proposed)", + pkgRevPublished: "Package Revision Approve/Publish", + pkgRevWaitPublished: "Package Revision Wait Published", + pkgRevProposeDeletion: "Package Revision Propose Deletion", + pkgRevDelete: "Package Revision Delete", +} + +func (v PorchAPIVersion) RepoOperations() []string { + return append([]string(nil), repoOperations...) +} + +func (v PorchAPIVersion) PkgRevOperations() []string { + switch v { + case PorchAPIV1Alpha2: + return append([]string(nil), v1alpha2PkgRevOperations...) + default: + return append([]string(nil), v1alpha1PkgRevOperations...) + } +} + +func (v PorchAPIVersion) AllOperations() []string { + return append(v.RepoOperations(), v.PkgRevOperations()...) +} + +func (v PorchAPIVersion) LifecyclePkgRevOperations() []string { + ops := v.PkgRevOperations() + lifecycle := make([]string, 0, len(ops)) + for _, op := range ops { + if op != pkgRevProposeDeletion && op != pkgRevDelete { + lifecycle = append(lifecycle, op) + } + } + return lifecycle +} + +func operationHeading(opKey string) string { + if heading := operationHeadings[opKey]; heading != "" { + return heading + } + return opKey +} + +func isDeletionOperation(opKey string) bool { + return opKey == pkgRevProposeDeletion || opKey == pkgRevDelete +} diff --git a/test/performance/performance_suite.go b/test/performance/performance_suite.go index ffdd0fd1a..c9aae6d46 100644 --- a/test/performance/performance_suite.go +++ b/test/performance/performance_suite.go @@ -33,6 +33,7 @@ import ( "github.com/joho/godotenv" porchclient "github.com/kptdev/porch/api/generated/clientset/versioned" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" + porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" "github.com/kptdev/porch/internal/telemetry" pkgerrors "github.com/pkg/errors" @@ -44,7 +45,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" ) @@ -55,9 +55,10 @@ var ( scheme = runtime.NewScheme() namespace = flag.String("namespace", "porch-metrics", "Kubernetes namespace to use for the test") + porchAPIVersion = flag.String("api-version", string(PorchAPIV1Alpha1), "Porch API version to test (v1alpha1 or v1alpha2)") numRepos = flag.Int("repos", 1, "Number of repositories to create") - numPackages = flag.Int("packages", 5, "Number of packages per repository") - numRevisions = flag.Int("revisions", 5, "Number of package revisions per package") + numPackages = flag.Int("packages", 1, "Number of packages per repository") + numRevisions = flag.Int("revisions", 1, "Number of package revisions per package") repoParallelism = flag.Int("repo-parallelism", 1, "Number of repositories to create in parallel") packageParallelism = flag.Int("package-parallelism", 1, "Number of packages to create in parallel per repository") errorRate = flag.Float64("error-rate", 0.1, "Maximum percentage of package revisions allowed to fail lifecycle transition") @@ -70,7 +71,7 @@ var ( lifecycleCSV = flag.String("repo-results-csv", "load_test_lifecycle_results.csv", "File name for repository results CSV") operationsCSV = flag.String("operations-csv", "load_test_operations_results.csv", "File name for operations details CSV") deletionCSV = flag.String("deletion-csv", "load_test_deletion_results.csv", "File name for deletion operations CSV") - packagePath = flag.String("package-path", "packages/large-package", "Path to the directory containing package resources") + packagePath = flag.String("package-path", "packages/small-package", "Path to the directory containing package resources") giteaURL = flag.String("gitea-url", "http://localhost:3000", "Base URL for the Gitea API") giteaUsername = flag.String("gitea-username", "porch", "Gitea username") @@ -97,6 +98,7 @@ type PerfTestSuite struct { resultsLogger *ResultsLogger otelResources *telemetry.OTelResources enablePrometheus bool + lifecycleDriver LifecycleDriver metrics map[string]TestMetrics metricsMutex sync.RWMutex @@ -107,6 +109,7 @@ type PerfTestSuite struct { } type TestOptions struct { + apiVersion PorchAPIVersion namespace string numRepos int numPkgs int @@ -138,6 +141,7 @@ type CSVOptions struct { func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(porchapi.AddToScheme(scheme)) + utilruntime.Must(porchv1alpha2.AddToScheme(scheme)) utilruntime.Must(configapi.AddToScheme(scheme)) } @@ -174,6 +178,12 @@ func (t *PerfTestSuite) recordPkgRevMetric(repoName, pkgName string, revisionNum Metrics: make(map[string]OperationMetrics), } } + if existing, ok := pkgRevEntry.Metrics[opKey]; ok && existing.Error == nil && op.Error == nil { + op.Duration += existing.Duration + if op.Timestamp.After(existing.Timestamp) { + op.Timestamp = existing.Timestamp + } + } pkgRevEntry.Metrics[opKey] = op repoMetrics.pkgRevMetrics[pkgName][revisionNum] = pkgRevEntry t.metrics[repoName] = repoMetrics @@ -204,8 +214,14 @@ func (t *PerfTestSuite) SetupSuite() { flag.Parse() + apiVersion, err := ParsePorchAPIVersion(*porchAPIVersion) + if err != nil { + t.T().Fatalf("Invalid api-version flag: %v", err) + } + t.metrics = make(map[string]TestMetrics) t.testOptions = TestOptions{ + apiVersion: apiVersion, namespace: *namespace, numRepos: *numRepos, numPkgs: *numPackages, @@ -234,7 +250,7 @@ func (t *PerfTestSuite) SetupSuite() { deletionCSV: *deletionCSV, } - logger, err := t.NewTestLogger(t.logOptions.metricsLogFile) + logger, err := t.NewTestLogger(t.logOptions.metricsLogFile, apiVersion) if err != nil { t.T().Fatalf("Failed to create logger: %v", err) } @@ -243,6 +259,7 @@ func (t *PerfTestSuite) SetupSuite() { if err != nil { t.T().Fatalf("Failed to create results logger: %v", err) } + resultsLogger.LogTestConfig(apiVersion, t.testOptions, t.enablePrometheus) cfg, err := config.GetConfig() if err != nil { @@ -267,6 +284,12 @@ func (t *PerfTestSuite) SetupSuite() { t.resultsLogger = resultsLogger t.enablePrometheus = *enablePrometheus + lifecycleDriver, err := NewLifecycleDriver(apiVersion, t) + if err != nil { + t.T().Fatalf("Failed to create lifecycle driver: %v", err) + } + t.lifecycleDriver = lifecycleDriver + if t.enablePrometheus { if err := os.Setenv("OTEL_METRICS_EXPORTER", "prometheus"); err != nil { t.T().Fatalf("Failed to set OTEL_METRICS_EXPORTER: %v", err) @@ -280,10 +303,11 @@ func (t *PerfTestSuite) SetupSuite() { } t.otelResources = otelRes t.T().Logf("OTel metrics server started on port %v", prometheusPort) - telemetry.PerfTestSetTestRunInfo("porch-performance-test", t.testOptions.namespace, time.Now()) + telemetry.PerfTestSetTestRunInfo("porch-performance-test", t.testOptions.namespace, string(apiVersion), time.Now()) } t.T().Logf(" Running load test with:") + t.T().Logf(" API version: %s", apiVersion) t.T().Logf(" Namespace: %s", t.testOptions.namespace) t.T().Logf(" %d repositories", t.testOptions.numRepos) t.T().Logf(" %d packages per repository", t.testOptions.numPkgs) @@ -442,10 +466,7 @@ func (t *PerfTestSuite) createAndSetupRepo(repoName string) { Error: err, Timestamp: start, }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(giteaRepoCreate, repoName, "", duration, err) - } + t.recordPerfMetric(giteaRepoCreate, repoName, "", duration, err) if err != nil { t.T().Errorf("Failed to create Gitea repository: %v", err) @@ -470,6 +491,9 @@ func (t *PerfTestSuite) createAndSetupRepo(repoName string) { }, }, } + if annotations := t.lifecycleDriver.RepositoryAnnotations(); len(annotations) > 0 { + repo.Annotations = annotations + } err = t.client.Create(t.ctx, repo) duration = time.Since(start) @@ -480,10 +504,7 @@ func (t *PerfTestSuite) createAndSetupRepo(repoName string) { Error: err, Timestamp: start, }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(porchRepoCreate, repoName, "", duration, err) - } + t.recordPerfMetric(porchRepoCreate, repoName, "", duration, err) if err != nil { t.T().Errorf("Failed to create Porch repository: %v", err) @@ -503,10 +524,7 @@ func (t *PerfTestSuite) createAndSetupRepo(repoName string) { Error: err, Timestamp: start, }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(repoWait, repoName, "", duration, err) - } + t.recordPerfMetric(repoWait, repoName, "", duration, err) } func createGiteaRepo(ctx context.Context, opts TestOptions, repoName string) error { @@ -568,6 +586,7 @@ func deleteGiteaRepo(ctx context.Context, opts TestOptions, repoName string) err return nil } + func (t *PerfTestSuite) waitForRepository(name string, timeout time.Duration) error { start := time.Now() for { @@ -610,189 +629,6 @@ func (t *PerfTestSuite) waitForRepository(name string, timeout time.Duration) er } } -func (t *PerfTestSuite) doLifecycle(repoName, pkgName string, revisionNum int) (string, error) { - var list porchapi.PackageRevisionList - var taskList []porchapi.Task - - t.initPkgRevMetrics(repoName, pkgName, revisionNum) - - start := time.Now() - err := retry.RetryOnConflict(retryBackoff, func() error { - return t.client.List(t.ctx, &list, client.InNamespace(t.testOptions.namespace)) - }) - duration := time.Since(start) - - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevList, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevList, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevList, repoName, pkgName, duration, err) - } - - if err != nil { - return "", err - } - - var latestPR *porchapi.PackageRevision - for i := range list.Items { - pr := &list.Items[i] - if pr.Spec.PackageName == pkgName && - pr.Spec.RepositoryName == repoName && - pr.Spec.Lifecycle == porchapi.PackageRevisionLifecyclePublished { - if latestPR == nil || pr.Spec.Revision > latestPR.Spec.Revision { - latestPR = pr - } - } - } - - if revisionNum == 1 { - taskList = []porchapi.Task{ - { - Type: porchapi.TaskTypeInit, - Init: &porchapi.PackageInitTaskSpec{ - Description: fmt.Sprintf("Test package %s for Porch metrics", pkgName), - Keywords: []string{"test", "metrics"}, - Site: "https://nephio.org", - }, - }, - } - if t.enablePrometheus { - telemetry.PerfTestIncrementPackageCounter() - } - } else if latestPR != nil { - taskList = []porchapi.Task{ - { - Type: porchapi.TaskTypeEdit, - Edit: &porchapi.PackageEditTaskSpec{ - Source: &porchapi.PackageRevisionRef{ - Name: latestPR.Name, - }, - }, - }, - } - } - - workspace := fmt.Sprintf("v%d", revisionNum) - pkgRev := &porchapi.PackageRevision{ - TypeMeta: metav1.TypeMeta{ - Kind: "PackageRevision", - APIVersion: porchapi.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: t.testOptions.namespace, - }, - Spec: porchapi.PackageRevisionSpec{ - PackageName: pkgName, - WorkspaceName: workspace, - RepositoryName: repoName, - Tasks: taskList, - }, - } - - if err = t.createPackageRevision(pkgRev, repoName, revisionNum); err != nil { - return "", err - } - - if err = t.updateOrCreatePackageRevisionResources(repoName, pkgName, pkgRev.Name, revisionNum); err != nil { - return "", err - } - - if err = t.proposeAndApprovePackage(repoName, pkgName, pkgRev.Name, revisionNum); err != nil { - return "", err - } - - return pkgRev.Name, nil -} - -func (t *PerfTestSuite) createPackageRevision(pkgRev *porchapi.PackageRevision, repoName string, revisionNum int) error { - start := time.Now() - if t.enablePrometheus { - telemetry.PerfTestRecordActiveOperation(pkgRevCreate, 1) - defer telemetry.PerfTestRecordActiveOperation(pkgRevCreate, -1) - } - - err := retry.RetryOnConflict(retryBackoff, func() error { - return t.client.Create(t.ctx, pkgRev) - }) - duration := time.Since(start) - - t.recordPkgRevMetric(repoName, pkgRev.Spec.PackageName, revisionNum, pkgRevCreate, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevCreate, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevCreate, repoName, pkgRev.Spec.PackageName, duration, err) - telemetry.PerfTestRecordPackageRevision(pkgRevCreate, err) - } - - if err != nil { - return err - } - - return nil -} - -func (t *PerfTestSuite) updateOrCreatePackageRevisionResources(repoName, pkgName, pkgRevName string, revisionNum int) error { - var resources porchapi.PackageRevisionResources - - start := time.Now() - err := retry.RetryOnConflict(retryBackoff, func() error { - return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &resources) - }) - duration := time.Since(start) - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevResourcesGet, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevResourcesGet, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevResourcesGet, repoName, pkgName, duration, err) - } - - if err != nil { - return err - } - - pkgResources := t.createPackageResources() - if resources.Spec.Resources == nil { - resources.Spec.Resources = make(map[string]string) - } - for name, content := range pkgResources { - resources.Spec.Resources[name] = content - } - - start = time.Now() - err = retry.RetryOnConflict(retryBackoff, func() error { - return t.client.Update(t.ctx, &resources) - }) - duration = time.Since(start) - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevUpdate, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevUpdate, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevUpdate, repoName, pkgName, duration, err) - } - - if err != nil { - return err - } - - return nil -} - func (t *PerfTestSuite) createPackageResources() map[string]string { resources := t.readPackageResources(t.testOptions.packagePath) if kptfile, ok := resources["Kptfile"]; ok { @@ -829,157 +665,3 @@ func (t *PerfTestSuite) readPackageResources(dir string) map[string]string { } return resources } - -func (t *PerfTestSuite) proposeAndApprovePackage(repoName, pkgName, pkgRevName string, revisionNum int) error { - var pkg porchapi.PackageRevision - - start := time.Now() - err := retry.RetryOnConflict(retryBackoff, func() error { - return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg) - }) - duration := time.Since(start) - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevGet, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevGet, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevGet, repoName, pkgName, duration, err) - } - - if err != nil { - return err - } - - start = time.Now() - initialLifecycle := pkg.Spec.Lifecycle - err = retry.RetryOnConflict(retryBackoff, func() error { - if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg); err != nil { - return err - } - pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecycleProposed - return t.client.Update(t.ctx, &pkg) - }) - duration = time.Since(start) - - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevPropose, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevPropose, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevPropose, repoName, pkgName, duration, err) - telemetry.PerfTestRecordLifecycleTransition(string(initialLifecycle), string(porchapi.PackageRevisionLifecycleProposed), repoName, pkgName, duration, err) - } - - if err != nil { - return err - } - - start = time.Now() - err = retry.RetryOnConflict(retryBackoff, func() error { - return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg) - }) - duration = time.Since(start) - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevGetProposed, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevGetProposed, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevGetProposed, repoName, pkgName, duration, err) - } - - if err != nil { - return err - } - - start = time.Now() - err = retry.RetryOnConflict(retryBackoff, func() error { - if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkg); err != nil { - return err - } - pkg.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished - _, err := t.clientSet.PorchV1alpha1().PackageRevisions(t.testOptions.namespace).UpdateApproval(t.ctx, pkgRevName, &pkg, metav1.UpdateOptions{}) - return err - }) - duration = time.Since(start) - - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevPublished, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevPublished, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevPublished, repoName, pkgName, duration, err) - telemetry.PerfTestRecordLifecycleTransition(string(porchapi.PackageRevisionLifecycleProposed), string(porchapi.PackageRevisionLifecyclePublished), repoName, pkgName, duration, err) - } - - return nil -} - -func (t *PerfTestSuite) deletePackageRevision(repoName, pkgName, pkgRevName string, revisionNum int) error { - var pkgRev porchapi.PackageRevision - err := retry.RetryOnConflict(retryBackoff, func() error { - return t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkgRev) - }) - if err != nil { - return err - } - - start := time.Now() - initialLifecycle := pkgRev.Spec.Lifecycle - err = retry.RetryOnConflict(retryBackoff, func() error { - if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkgRev); err != nil { - return err - } - pkgRev.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed - return t.client.Update(t.ctx, &pkgRev) - }) - duration := time.Since(start) - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevProposeDeletion, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevProposeDeletion, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevProposeDeletion, repoName, pkgName, duration, err) - telemetry.PerfTestRecordLifecycleTransition(string(initialLifecycle), string(porchapi.PackageRevisionLifecycleDeletionProposed), repoName, pkgName, duration, err) - } - - if err != nil { - return err - } - - start = time.Now() - err = retry.RetryOnConflict(retryBackoff, func() error { - if err := t.client.Get(t.ctx, client.ObjectKey{Namespace: t.testOptions.namespace, Name: pkgRevName}, &pkgRev); err != nil { - return err - } - return t.client.Delete(t.ctx, &pkgRev) - }) - duration = time.Since(start) - t.recordPkgRevMetric(repoName, pkgName, revisionNum, pkgRevDelete, OperationMetrics{ - Operation: fmt.Sprintf("%s:%d", pkgRevDelete, revisionNum), - Duration: duration, - Error: err, - Timestamp: start, - }) - - if t.enablePrometheus { - telemetry.PerfTestRecordMetric(pkgRevDelete, repoName, pkgName, duration, err) - telemetry.PerfTestRecordLifecycleTransition(string(porchapi.PackageRevisionLifecycleDeletionProposed), "deleted", repoName, pkgName, duration, err) - } - - return nil -} diff --git a/test/performance/porch_metrics_test.go b/test/performance/porch_metrics_test.go index eac9670ed..ac8b03d81 100644 --- a/test/performance/porch_metrics_test.go +++ b/test/performance/porch_metrics_test.go @@ -19,14 +19,11 @@ import ( "os" "path/filepath" "sort" - "strings" "sync" "testing" "time" - porchapi "github.com/kptdev/porch/api/porch/v1alpha1" "github.com/stretchr/testify/suite" - "sigs.k8s.io/controller-runtime/pkg/client" ) type PerformanceTests struct { @@ -142,65 +139,55 @@ func (t *PerformanceTests) TestIncreasePRsPerformance() { } func (t *PerformanceTests) deleteEnv(deletionStartTime *time.Time, deletionDuration *time.Duration, deletedCount *int) { - *deletionStartTime = time.Now() t.T().Log("\n=== Starting Deletion Operations ===") t.T().Logf("Deletion enabled: will delete all %d package revisions across %d repositories", t.testOptions.numRepos*t.testOptions.numPkgs*t.testOptions.numRevs, t.testOptions.numRepos) - var prList porchapi.PackageRevisionList - if err := t.client.List(t.ctx, &prList, client.InNamespace(t.testOptions.namespace)); err != nil { + candidates, err := t.lifecycleDriver.ListPackageRevisionsForDeletion() + if err != nil { t.T().Logf("failed to list package revisions for deletion: %v", err) - } else { - t.T().Logf("found %d package revisions to delete", len(prList.Items)) + return + } - sort.Slice(prList.Items, func(i, j int) bool { - var ri, rj int - if ws := prList.Items[i].Spec.WorkspaceName; strings.Contains(ws, "v") { - _, _ = fmt.Sscanf(ws, "v%d", &ri) - } - if ws := prList.Items[j].Spec.WorkspaceName; strings.Contains(ws, "v") { - _, _ = fmt.Sscanf(ws, "v%d", &rj) - } - return ri < rj - }) + t.T().Logf("found %d package revisions to delete", len(candidates)) - *deletedCount = 0 - for _, pr := range prList.Items { - if err := t.ctx.Err(); err != nil { - t.T().Logf("Deletion interrupted: %v", err) - break - } - prefix := fmt.Sprintf("%s-test-", t.testOptions.namespace) - if !strings.HasPrefix(pr.Spec.RepositoryName, prefix) { - continue - } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].RepoName != candidates[j].RepoName { + return candidates[i].RepoName < candidates[j].RepoName + } + if candidates[i].PackageName != candidates[j].PackageName { + return candidates[i].PackageName < candidates[j].PackageName + } + return candidates[i].RevisionNum > candidates[j].RevisionNum + }) - revisionNum := 1 - if strings.Contains(pr.Spec.WorkspaceName, "v") { - _, _ = fmt.Sscanf(pr.Spec.WorkspaceName, "v%d", &revisionNum) - } + *deletedCount = 0 + for _, pr := range candidates { + if err := t.ctx.Err(); err != nil { + t.T().Logf("Deletion interrupted: %v", err) + break + } - t.T().Logf("Deleting package revision: %s (repo: %s, pkg: %s, revision: %d)", - pr.Name, pr.Spec.RepositoryName, pr.Spec.PackageName, revisionNum) + t.T().Logf("Deleting package revision: %s (repo: %s, pkg: %s, revision: %d)", + pr.Name, pr.RepoName, pr.PackageName, pr.RevisionNum) - if err = t.deletePackageRevision(pr.Spec.RepositoryName, pr.Spec.PackageName, pr.Name, revisionNum); err == nil { - proposeDel := t.metrics[pr.Spec.RepositoryName].pkgRevMetrics[pr.Spec.PackageName][revisionNum].Metrics[pkgRevProposeDeletion] - del := t.metrics[pr.Spec.RepositoryName].pkgRevMetrics[pr.Spec.PackageName][revisionNum].Metrics[pkgRevDelete] - t.resultsLogger.LogDeleted(pr.Name, proposeDel.Duration+del.Duration) - *deletedCount++ - } else { - t.T().Errorf("failed to delete package revision: %s (repo: %s, pkg: %s, revision: %d)", pr.Name, pr.Spec.RepositoryName, pr.Spec.PackageName, revisionNum) - } + if err = t.lifecycleDriver.DeletePackageRevision(pr.RepoName, pr.PackageName, pr.Name, pr.RevisionNum); err == nil { + proposeDel := t.metrics[pr.RepoName].pkgRevMetrics[pr.PackageName][pr.RevisionNum].Metrics[pkgRevProposeDeletion] + del := t.metrics[pr.RepoName].pkgRevMetrics[pr.PackageName][pr.RevisionNum].Metrics[pkgRevDelete] + t.resultsLogger.LogDeleted(pr.Name, proposeDel.Duration+del.Duration) + *deletedCount++ + } else { + t.T().Errorf("failed to delete package revision: %s (repo: %s, pkg: %s, revision: %d)", pr.Name, pr.RepoName, pr.PackageName, pr.RevisionNum) } - *deletionDuration = time.Since(*deletionStartTime) - - t.T().Logf("Completed deletion of %d package revisions", deletedCount) } + *deletionDuration = time.Since(*deletionStartTime) + + t.T().Logf("Completed deletion of %d package revisions", *deletedCount) } func (t *PerformanceTests) printTestResults(logger *TestLogger) { - header := "\n=== Consolidated Performance Test Results ===" + header := fmt.Sprintf("\n=== Consolidated Performance Test Results (%s) ===", t.testOptions.apiVersion) t.T().Log(header) logger.LogResult("%s", header) @@ -212,45 +199,11 @@ func (t *PerformanceTests) printTestResults(logger *TestLogger) { t.T().Log(divider) logger.LogResult("%s", divider) - operationStats := make(map[string]*Stats) - - operationHeadings := map[string]string{ - giteaRepoCreate: "Create Gitea Repository ", - porchRepoCreate: "Create Porch Repository ", - repoWait: "Repository Ready Wait", - pkgRevList: "Package Revision List", - pkgRevCreate: "Package Revision Create", - pkgRevResourcesGet: "Package Revision Get Resources", - pkgRevUpdate: "Package Revision Update", - pkgRevGet: "Package Revision Get", - pkgRevPropose: "Package Revision Propose", - pkgRevGetProposed: "Package Revision Get (Proposed)", - pkgRevPublished: "Package Revision Approve/Publish", - pkgRevProposeDeletion: "Package Revision Propose Deletion", - pkgRevDelete: "Package Revision Delete", - } - - repoOperations := []string{ - giteaRepoCreate, - porchRepoCreate, - repoWait, - } - - pkgRevOperations := []string{ - pkgRevList, - pkgRevCreate, - pkgRevResourcesGet, - pkgRevUpdate, - pkgRevGet, - pkgRevPropose, - pkgRevGetProposed, - pkgRevPublished, - pkgRevProposeDeletion, - pkgRevDelete, - } - - allOperations := append(repoOperations, pkgRevOperations...) + repoOperations := t.testOptions.apiVersion.RepoOperations() + pkgRevOperations := t.testOptions.apiVersion.PkgRevOperations() + allOperations := t.testOptions.apiVersion.AllOperations() + operationStats := make(map[string]*Stats, len(allOperations)) for _, op := range allOperations { operationStats[op] = &Stats{} } @@ -326,11 +279,7 @@ func (t *PerformanceTests) printTestResults(logger *TestLogger) { continue } - heading := operationHeadings[opKey] - if heading == "" { - heading = opKey - } - headingWithNum := fmt.Sprintf("%s R%d", heading, i) + headingWithNum := fmt.Sprintf("%s R%d", operationHeading(opKey), i) result := fmt.Sprintf("%-37s %-11v %-11v %-11v %-11v", headingWithNum, repoOp.Duration.Round(time.Millisecond), @@ -392,11 +341,7 @@ func (t *PerformanceTests) printTestResults(logger *TestLogger) { if revCount > 0 { avg := revStats.Total / time.Duration(revCount) - heading := operationHeadings[opKey] - if heading == "" { - heading = opKey - } - headingWithRev := fmt.Sprintf("%s v%d", heading, k) + headingWithRev := fmt.Sprintf("%s v%d", operationHeading(opKey), k) result := fmt.Sprintf("%-37s %-11v %-11v %-11v %-11v", headingWithRev, revStats.Min.Round(time.Millisecond), @@ -542,7 +487,7 @@ func (t *PerformanceTests) processRepository(repoIndex, numRevs int, errorCalcul return } t.T().Logf("Creating revision %d/%d for package %s", k, t.testOptions.numRevs, pkgName) - if pkgRevName, err := t.doLifecycle(repoName, pkgName, k); err == nil { + if pkgRevName, err := t.lifecycleDriver.DoLifecycle(repoName, pkgName, k); err == nil { t.metricsMutex.RLock() for _, op := range t.metrics[repoName].pkgRevMetrics[pkgName][k].Metrics { if op.Operation == fmt.Sprintf("%s:%d", pkgRevPublished, k) { diff --git a/test/performance/types.go b/test/performance/types.go index 0ff3850de..0c9969168 100644 --- a/test/performance/types.go +++ b/test/performance/types.go @@ -15,13 +15,27 @@ package metrics import ( + "fmt" "time" ) +// PorchAPIVersion identifies which Porch PackageRevision API the performance test uses. +type PorchAPIVersion string + +const ( + PorchAPIV1Alpha1 PorchAPIVersion = "v1alpha1" + PorchAPIV1Alpha2 PorchAPIVersion = "v1alpha2" +) + +// Repository-level operation metric keys. +const ( + giteaRepoCreate = "GITEA-REPO-CREATE" + porchRepoCreate = "PORCH-REPO-CREATE" + repoWait = "REPO-WAIT" +) + +// Package revision operation metric keys shared across API versions. const ( - giteaRepoCreate = "GITEA-REPO-CREATE" - porchRepoCreate = "PORCH-REPO-CREATE" - repoWait = "REPO-WAIT" pkgRevList = "LIST" pkgRevGet = "GET" pkgRevGetProposed = "GET-PROPOSED" @@ -34,6 +48,22 @@ const ( pkgRevDelete = "DELETE" ) +// v1alpha2 controller reconciliation operation metric keys. +const ( + pkgRevWaitReady = "WAIT-READY" + pkgRevWaitRendered = "WAIT-RENDERED" + pkgRevWaitPublished = "WAIT-PUBLISHED" +) + +func ParsePorchAPIVersion(version string) (PorchAPIVersion, error) { + switch PorchAPIVersion(version) { + case PorchAPIV1Alpha1, PorchAPIV1Alpha2: + return PorchAPIVersion(version), nil + default: + return "", fmt.Errorf("unsupported porch API version %q (supported: %s, %s)", version, PorchAPIV1Alpha1, PorchAPIV1Alpha2) + } +} + type OperationMetrics struct { Operation string Duration time.Duration @@ -52,9 +82,19 @@ type PackageRevisionMetrics struct { Revision int Metrics map[string]OperationMetrics } + type Stats struct { Min time.Duration Max time.Duration Total time.Duration Count int } + +// DeletionCandidate identifies a package revision targeted for deletion cleanup. +type DeletionCandidate struct { + Name string + RepoName string + PackageName string + WorkspaceName string + RevisionNum int +} From d74ab8a74ff7e4adece1ceb26536084c8fb56192 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Sat, 18 Jul 2026 01:54:46 +0200 Subject: [PATCH 04/13] Add metrics for PR controller and Git Push Signed-off-by: Rendre Greyling --- .../controllers/packagerevision/ownership.go | 5 + .../packagerevision_controller.go | 7 + .../pkg/controllers/packagerevision/render.go | 4 + .../grafana-porch-server-dashboard.json | 1157 +++++++++++++++-- internal/telemetry/metrics.go | 65 +- internal/telemetry/metrics_test.go | 33 + pkg/externalrepo/git/git.go | 22 +- pkg/registry/porch/packagerevision.go | 20 +- .../porch/packagerevision_approval.go | 8 +- .../porch/packagerevisionresources.go | 12 +- 10 files changed, 1163 insertions(+), 170 deletions(-) diff --git a/controllers/packagerevisions/pkg/controllers/packagerevision/ownership.go b/controllers/packagerevisions/pkg/controllers/packagerevision/ownership.go index 12fdc8ffd..d51c98522 100644 --- a/controllers/packagerevisions/pkg/controllers/packagerevision/ownership.go +++ b/controllers/packagerevisions/pkg/controllers/packagerevision/ownership.go @@ -17,9 +17,11 @@ package packagerevision import ( "context" "fmt" + "time" porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/repository" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -66,6 +68,9 @@ func (r *PackageRevisionReconciler) handleDeletion(ctx context.Context, pr *porc // shared content cache. "Not found" errors are treated as success — there // is nothing to clean up if the package or repo doesn't exist in the cache. func (r *PackageRevisionReconciler) deleteFromGit(ctx context.Context, pr *porchv1alpha2.PackageRevision) error { + start := time.Now() + defer telemetry.RecordControllerOperation(telemetry.ResourcePackageRevision, "DELETE", start) + repoKey := repository.RepositoryKey{ Namespace: pr.Namespace, Name: pr.Spec.RepositoryName, diff --git a/controllers/packagerevisions/pkg/controllers/packagerevision/packagerevision_controller.go b/controllers/packagerevisions/pkg/controllers/packagerevision/packagerevision_controller.go index 364a2739d..634290aa7 100644 --- a/controllers/packagerevisions/pkg/controllers/packagerevision/packagerevision_controller.go +++ b/controllers/packagerevisions/pkg/controllers/packagerevision/packagerevision_controller.go @@ -21,6 +21,7 @@ import ( porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" "github.com/kptdev/porch/controllers/functionconfigs" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/repository" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -160,7 +161,9 @@ func (r *PackageRevisionReconciler) reconcileLifecycle(ctx context.Context, pr * log.Info("lifecycle transition", "name", pr.Name, "current", current, "desired", desired) + start := time.Now() updated, err := r.ContentCache.UpdateLifecycle(ctx, repoKey, pr.Spec.PackageName, pr.Spec.WorkspaceName, desired) + telemetry.RecordControllerOperation(telemetry.ResourcePackageRevision, "UPDATE", start) if err != nil { log.Error(err, "lifecycle transition failed") r.updateStatus(ctx, pr, nil, "", readyCondition(pr.Generation, metav1.ConditionFalse, porchv1alpha2.ReasonFailed, err.Error())) @@ -201,6 +204,8 @@ func (r *PackageRevisionReconciler) reconcileSource(ctx context.Context, pr *por log := log.FromContext(ctx) log.Info("applying source", "type", creationSource, "name", pr.Name) + start := time.Now() + // TODO: CreateNewDraft always receives lifecycle=Draft — consider removing the lifecycle parameter from the interface. draft, err := r.ContentCache.CreateNewDraft(ctx, repoKey, pr.Spec.PackageName, pr.Spec.WorkspaceName, string(porchv1alpha2.PackageRevisionLifecycleDraft)) if err != nil { @@ -230,6 +235,8 @@ func (r *PackageRevisionReconciler) reconcileSource(ctx context.Context, pr *por ) r.ensureLatestRevisionLabel(ctx, pr) + telemetry.RecordControllerOperation(telemetry.ResourcePackageRevision, "CREATE", start) + result := ctrl.Result{Requeue: true} return &result, nil } diff --git a/controllers/packagerevisions/pkg/controllers/packagerevision/render.go b/controllers/packagerevisions/pkg/controllers/packagerevision/render.go index 16dab2825..04e529ed8 100644 --- a/controllers/packagerevisions/pkg/controllers/packagerevision/render.go +++ b/controllers/packagerevisions/pkg/controllers/packagerevision/render.go @@ -19,6 +19,7 @@ import ( "fmt" iofs "io/fs" "strings" + "time" fnresult "github.com/kptdev/kpt/api/fnresult/v1" kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" @@ -27,6 +28,7 @@ import ( "github.com/kptdev/kpt/pkg/lib/kptops" "github.com/kptdev/kpt/pkg/lib/runneroptions" porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/repository" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -172,6 +174,8 @@ func isPushOnRenderFailure(pr *porchv1alpha2.PackageRevision) bool { // executeRender performs the render, handles failure/stale, and writes results. func (r *PackageRevisionReconciler) executeRender(ctx context.Context, pr *porchv1alpha2.PackageRevision, repoKey repository.RepositoryKey) (*ctrl.Result, error) { log := log.FromContext(ctx) + start := time.Now() + defer telemetry.RecordControllerOperation(telemetry.ResourcePackageRevisionResources, "UPDATE", start) resources, err := r.readPackageResources(ctx, repoKey, pr.Spec.PackageName, pr.Spec.WorkspaceName) if err != nil { diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json index e1f2db3cd..9e45d1209 100644 --- a/deployments/metrics-resources/grafana-porch-server-dashboard.json +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -1,5 +1,7 @@ { - "annotations": { "list": [] }, + "annotations": { + "list": [] + }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -9,42 +11,98 @@ "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 100, "panels": [], "title": "PackageRevisions API Calls", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions GET latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, "id": 101, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"GET\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"GET\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"GET\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"GET\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"GET\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"GET\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"GET\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"GET\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -53,35 +111,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions LIST latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, "id": 102, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"LIST\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"LIST\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"LIST\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"LIST\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"LIST\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"LIST\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"LIST\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"LIST\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -90,35 +199,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions CREATE latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, "id": 103, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"CREATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"CREATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"CREATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"CREATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"CREATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"CREATE\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"CREATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"CREATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -127,35 +287,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions UPDATE latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, "id": 104, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"UPDATE\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -164,35 +375,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions DELETE latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, "id": 105, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"DELETE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"DELETE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"DELETE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevision\", verb=\"DELETE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"DELETE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"DELETE\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevision\", verb=\"DELETE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevision\", verb=\"DELETE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -202,42 +464,98 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, "id": 200, "panels": [], "title": "PackageRevisionResources API Calls", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisionResources GET latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, "id": 201, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"GET\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"GET\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"GET\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"GET\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"GET\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -246,35 +564,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisionResources UPDATE latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, "id": 202, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"UPDATE\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -283,35 +652,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisionResources LIST latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, "id": 203, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"LIST\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionResources\", verb=\"LIST\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"LIST\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionResources\", verb=\"LIST\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionResources\", verb=\"LIST\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -321,42 +741,98 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, "id": 300, "panels": [], "title": "Approval API Calls", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions Approval GET latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, "id": 301, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"GET\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"GET\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionApproval\", verb=\"GET\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionApproval\", verb=\"GET\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionApproval\", verb=\"GET\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -365,35 +841,86 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "PackageRevisions Approval UPDATE latency metrics (p95, p99, average).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, "id": 302, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p95", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m])) by (le))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"PackageRevisionApproval\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) by (le))", "legendFormat": "p99", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionApproval\", verb=\"UPDATE\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"PackageRevisionApproval\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"PackageRevisionApproval\", verb=\"UPDATE\",api_version=\"$api_version\"}[5m]))", "legendFormat": "avg", "refId": "C" } @@ -403,30 +930,80 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, "id": 500, "panels": [], "title": "Request Counts by User", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "Rate of PackageRevision requests per operation and user.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, "id": 501, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "sum(rate(porch_api_requests_by_user_total{resource=\"PackageRevision\"}[5m])) by (op, user)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_requests_by_user_total{resource=\"PackageRevision\",api_version=\"$api_version\"}[5m])) by (op, user)", "legendFormat": "{{op}} / {{user}}", "refId": "A" } @@ -435,22 +1012,67 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "Rate of ExternalRepo requests per operation and user.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 52 + }, "id": 502, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "sum(rate(porch_api_requests_by_user_total{resource=\"ExternalRepo\"}[5m])) by (op, user)", "legendFormat": "{{op}} / {{user}}", "refId": "A" @@ -460,41 +1082,298 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "description": "Total cumulative request counts broken down by resource, operation, and user.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 60 + }, "id": 503, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "expr": "sum(porch_api_requests_by_user_total) by (resource, op, user)", "legendFormat": "{{resource}} / {{op}} / {{user}}", "refId": "A" } ], - "title": "All Resources — Total Request Count by Resource, Operation & User", + "title": "All Resources \u2014 Total Request Count by Resource, Operation & User", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 600, + "panels": [], + "title": "ExternalRepo Git Operations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "ExternalRepo FETCH latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 601, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"ExternalRepo\", verb=\"FETCH\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"ExternalRepo\", verb=\"FETCH\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"ExternalRepo\", verb=\"FETCH\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"ExternalRepo\", verb=\"FETCH\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "ExternalRepo FETCH", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "ExternalRepo PUSH latency metrics (p95, p99, average).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 602, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"ExternalRepo\", verb=\"PUSH\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(porch_api_call_duration_seconds_bucket{resource=\"ExternalRepo\", verb=\"PUSH\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(porch_api_call_duration_seconds_sum{resource=\"ExternalRepo\", verb=\"PUSH\"}[5m])) / sum(rate(porch_api_call_duration_seconds_count{resource=\"ExternalRepo\", verb=\"PUSH\"}[5m]))", + "legendFormat": "avg", + "refId": "C" + } + ], + "title": "ExternalRepo PUSH", "type": "timeseries" } ], "refresh": "10s", "schemaVersion": 38, "style": "dark", - "tags": ["porch", "api"], - "templating": { "list": [] }, - "time": { "from": "now-1h", "to": "now" }, + "tags": [ + "porch", + "api" + ], + "templating": { + "list": [ + { + "name": "api_version", + "type": "query", + "query": "label_values(porch_api_call_duration_seconds, api_version)", + "current": { + "text": "v1alpha1", + "value": "v1alpha1" + }, + "refresh": 1, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, "timepicker": {}, "timezone": "", "title": "Porch API", "uid": "porch-api-queues", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index 17c6a32e5..d594a3dc4 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -29,6 +29,16 @@ import ( const meterName = "github.com/kptdev/porch" +const ( + APIVersionV1Alpha1 = "v1alpha1" + APIVersionV1Alpha2 = "v1alpha2" + ControllerUser = "packagerevision-controller" + + ResourcePackageRevision = "PackageRevision" + ResourcePackageRevisionResources = "PackageRevisionResources" + ResourceExternalRepo = "ExternalRepo" +) + var ( apiCallDurationSeconds metric.Float64Histogram RequestsTotal metric.Float64Counter @@ -164,7 +174,7 @@ func InitMetrics() (err error) { } // Porch server and function runner metric recording functions -func RecordAPICallDuration(resource, verb string, durationSeconds float64) { +func RecordAPICallDuration(resource, verb, apiVersion string, durationSeconds float64) { if apiCallDurationSeconds == nil { return } @@ -172,20 +182,69 @@ func RecordAPICallDuration(resource, verb string, durationSeconds float64) { metric.WithAttributes( attribute.String("resource", resource), attribute.String("verb", verb), + attribute.String("api_version", apiVersion), ), ) } -func RecordRequestCount(ctx context.Context, resource, op string) { +func RecordRequestCount(ctx context.Context, resource, op, apiVersion string) { if RequestsTotal == nil { return } - user := getK8sUserName(ctx) + recordRequestCount(resource, op, apiVersion, getK8sUserName(ctx)) +} + +func RecordControllerRequestCount(resource, op, apiVersion string) { + if RequestsTotal == nil { + return + } + recordRequestCount(resource, op, apiVersion, ControllerUser) +} + +// RecordControllerOperation records duration and request count for a v1alpha2 controller operation. +func RecordControllerOperation(resource, verb string, start time.Time) { + RecordAPICallDuration(resource, verb, APIVersionV1Alpha2, time.Since(start).Seconds()) + RecordControllerRequestCount(resource, verb, APIVersionV1Alpha2) +} + +func recordRequestCount(resource, op, apiVersion, user string) { RequestsTotal.Add(context.Background(), 1, metric.WithAttributes( attribute.String("resource", resource), attribute.String("op", op), attribute.String("user", user), + attribute.String("api_version", apiVersion), + ), + ) +} + +// External git operations are shared infrastructure and are not tagged with api_version. +func RecordExternalRepoOperation(ctx context.Context, op string, start time.Time) { + recordExternalRepoDuration(op, time.Since(start).Seconds()) + RecordExternalRepoRequestCount(ctx, op) +} + +func recordExternalRepoDuration(op string, durationSeconds float64) { + if apiCallDurationSeconds == nil { + return + } + apiCallDurationSeconds.Record(context.Background(), durationSeconds, + metric.WithAttributes( + attribute.String("resource", ResourceExternalRepo), + attribute.String("verb", op), + ), + ) +} + +func RecordExternalRepoRequestCount(ctx context.Context, op string) { + if RequestsTotal == nil { + return + } + RequestsTotal.Add(context.Background(), 1, + metric.WithAttributes( + attribute.String("resource", ResourceExternalRepo), + attribute.String("op", op), + attribute.String("user", getK8sUserName(ctx)), ), ) } diff --git a/internal/telemetry/metrics_test.go b/internal/telemetry/metrics_test.go index fb15edb52..4b421538c 100644 --- a/internal/telemetry/metrics_test.go +++ b/internal/telemetry/metrics_test.go @@ -17,6 +17,7 @@ package telemetry import ( "context" "testing" + "time" "github.com/kptdev/porch/pkg/repository" "github.com/stretchr/testify/assert" @@ -52,6 +53,38 @@ func TestRecordPackageRevisionResourcesSize_NilInstruments(t *testing.T) { assert.NotPanics(t, func() { RecordPackageRevisionResourcesSize(context.Background(), fake, 1024) }) } +func TestRecordAPICallDuration_IncludesAPIVersion(t *testing.T) { + previousMp := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(mp) + defer func() { + otel.SetMeterProvider(previousMp) + mp.Shutdown(context.Background()) + }() + + require.NoError(t, InitMetrics()) + + RecordControllerOperation(ResourcePackageRevision, "UPDATE", time.Now().Add(-time.Millisecond)) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + + var foundDuration, foundRequest bool + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch m.Name { + case "porch_api_call_duration_seconds": + foundDuration = true + case "porch_api_requests_by_user": + foundRequest = true + } + } + } + assert.True(t, foundDuration, "expected porch_api_call_duration_seconds to be recorded") + assert.True(t, foundRequest, "expected porch_api_requests_by_user to be recorded") +} + func TestRecordPackageRevisionResourcesSize_RecordsMetrics(t *testing.T) { previousMp := otel.GetMeterProvider() reader := sdkmetric.NewManualReader() diff --git a/pkg/externalrepo/git/git.go b/pkg/externalrepo/git/git.go index 000068046..5374008e3 100644 --- a/pkg/externalrepo/git/git.go +++ b/pkg/externalrepo/git/git.go @@ -58,8 +58,6 @@ const ( fileName = "README.md" commitMessage = "Initial commit: Creating main branch" - telemetryName = "ExternalRepo" - // Retry delay constants baseRetryDelay = 200 * time.Millisecond hookRetryDelay = 1 * time.Second @@ -647,7 +645,7 @@ func (r *gitRepository) DeletePackageRevision(ctx context.Context, pr2Delete rep return fmt.Errorf("cannot delete package with the ref name %s", referenceName) } - if err := r.pushAndCleanup(ctx, refSpecs, commitOps); err != nil { + if err := r.pushPackageRevisions(ctx, refSpecs, commitOps); err != nil { if pkgerrors.Is(err, git.NoErrAlreadyUpToDate) { klog.Infof("All remote references are already up to date for deleting package %s", pr2Delete.Key()) } else { @@ -1133,10 +1131,11 @@ func (r *gitRepository) fetchRemoteRepository(ctx context.Context) error { ctx, span := tracer.Start(ctx, "gitRepository::fetchRemoteRepository", trace.WithAttributes()) defer span.End() - telemetry.RecordRequestCount(ctx, telemetryName, "FETCH") - start := time.Now() - defer func() { klog.V(2).Infof("Fetching repository %q took %s", r.key.Name, time.Since(start)) }() + defer func() { + telemetry.RecordExternalRepoOperation(ctx, "FETCH", start) + klog.V(2).Infof("Fetching repository %q took %s", r.key.Name, time.Since(start)) + }() if ctx.Err() != nil { return ctx.Err() @@ -1407,6 +1406,13 @@ func (r *gitRepository) executeCommitOperations(ctx context.Context, repo *git.R return nil } +func (r *gitRepository) pushPackageRevisions(ctx context.Context, ph *pushRefSpecBuilder, commitOps *commitOperationBuilder) error { + start := time.Now() + err := r.pushAndCleanup(ctx, ph, commitOps) + telemetry.RecordExternalRepoOperation(ctx, "PUSH", start) + return err +} + func (r *gitRepository) pushAndCleanup(ctx context.Context, ph *pushRefSpecBuilder, commitOps *commitOperationBuilder) error { ctx, span := tracer.Start(ctx, "gitRepository::pushAndCleanup") defer span.End() @@ -1764,7 +1770,7 @@ func (r *gitRepository) UpdateLifecycle(ctx context.Context, pkgRev *gitPackageR } r.mutex.Unlock() // Release mutex before git operations - if err := r.pushAndCleanup(ctx, refSpecs, nil); err != nil { + if err := r.pushPackageRevisions(ctx, refSpecs, nil); err != nil { if !pkgerrors.Is(err, git.NoErrAlreadyUpToDate) { return err } @@ -1913,7 +1919,7 @@ func (r *gitRepository) ClosePackageRevisionDraft(ctx context.Context, prd repos return nil, fmt.Errorf("package has unrecognized lifecycle: %q", d.lifecycle) } - if err := d.repo.pushAndCleanup(ctx, refSpecs, commitOps); err != nil { + if err := d.repo.pushPackageRevisions(ctx, refSpecs, commitOps); err != nil { if !pkgerrors.Is(err, git.NoErrAlreadyUpToDate) { klog.Errorf("Failed to push package %s to %s: %v", d.Key().PkgKey.ToFullPathname(), targetBranch, err) return nil, err diff --git a/pkg/registry/porch/packagerevision.go b/pkg/registry/porch/packagerevision.go index 2d2a30e0f..094819069 100644 --- a/pkg/registry/porch/packagerevision.go +++ b/pkg/registry/porch/packagerevision.go @@ -79,10 +79,10 @@ func (r *packageRevisions) List(ctx context.Context, options *metainternalversio start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prTelemetryName, "LIST", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prTelemetryName, "LIST", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "LIST") + telemetry.RecordRequestCount(ctx, prTelemetryName, "LIST", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestID(ctx) @@ -128,10 +128,10 @@ func (r *packageRevisions) Get(ctx context.Context, name string, _ *metav1.GetOp start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prTelemetryName, "GET", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prTelemetryName, "GET", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "GET") + telemetry.RecordRequestCount(ctx, prTelemetryName, "GET", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -160,10 +160,10 @@ func (r *packageRevisions) Create(ctx context.Context, runtimeObject runtime.Obj start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prTelemetryName, "CREATE", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prTelemetryName, "CREATE", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "CREATE") + telemetry.RecordRequestCount(ctx, prTelemetryName, "CREATE", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestID(ctx) @@ -307,10 +307,10 @@ func (r *packageRevisions) Update(ctx context.Context, name string, objInfo rest start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prTelemetryName, "UPDATE", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "UPDATE") + telemetry.RecordRequestCount(ctx, prTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -337,10 +337,10 @@ func (r *packageRevisions) Delete(ctx context.Context, name string, deleteValida start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prTelemetryName, "DELETE", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prTelemetryName, "DELETE", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "DELETE") + telemetry.RecordRequestCount(ctx, prTelemetryName, "DELETE", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) diff --git a/pkg/registry/porch/packagerevision_approval.go b/pkg/registry/porch/packagerevision_approval.go index 231d8e80a..a76046560 100644 --- a/pkg/registry/porch/packagerevision_approval.go +++ b/pkg/registry/porch/packagerevision_approval.go @@ -60,10 +60,10 @@ func (a *packageRevisionApproval) Get(ctx context.Context, name string, _ *metav start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(praTelemetryName, "GET", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(praTelemetryName, "GET", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "GET") + telemetry.RecordRequestCount(ctx, prTelemetryName, "GET", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -83,10 +83,10 @@ func (a *packageRevisionApproval) Update(ctx context.Context, name string, objIn start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(praTelemetryName, "UPDATE", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(praTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "GET") + telemetry.RecordRequestCount(ctx, prTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) diff --git a/pkg/registry/porch/packagerevisionresources.go b/pkg/registry/porch/packagerevisionresources.go index 865ca7d7b..66824c96f 100644 --- a/pkg/registry/porch/packagerevisionresources.go +++ b/pkg/registry/porch/packagerevisionresources.go @@ -76,10 +76,10 @@ func (r *packageRevisionResources) List(ctx context.Context, options *metaintern start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prrTelemetryName, "LIST", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prrTelemetryName, "LIST", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prrTelemetryName, "LIST") + telemetry.RecordRequestCount(ctx, prrTelemetryName, "LIST", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestID(ctx) @@ -123,10 +123,10 @@ func (r *packageRevisionResources) Get(ctx context.Context, name string, _ *meta start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prrTelemetryName, "GET", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prrTelemetryName, "GET", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prrTelemetryName, "GET") + telemetry.RecordRequestCount(ctx, prrTelemetryName, "GET", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -157,10 +157,10 @@ func (r *packageRevisionResources) Update(ctx context.Context, name string, objI start := time.Now() defer func() { span.End() - telemetry.RecordAPICallDuration(prrTelemetryName, "UPDATE", time.Since(start).Seconds()) + telemetry.RecordAPICallDuration(prrTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prrTelemetryName, "UPDATE") + telemetry.RecordRequestCount(ctx, prrTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) From c153e64d053004d8ad5acd6caafc68f68dd567cf Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Sat, 18 Jul 2026 02:10:13 +0200 Subject: [PATCH 05/13] Add Jaeger for tracing Signed-off-by: Rendre Greyling --- deployments/metrics/jaeger-deployment.yaml | 82 ++++++++++++++++++++++ deployments/porch/2-function-runner.yaml | 6 +- deployments/porch/3-porch-server.yaml | 6 +- deployments/porch/9-controllers.yaml | 6 +- scripts/deploy-monitoring.sh | 29 ++++++-- 5 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 deployments/metrics/jaeger-deployment.yaml diff --git a/deployments/metrics/jaeger-deployment.yaml b/deployments/metrics/jaeger-deployment.yaml new file mode 100644 index 000000000..7b28f4f19 --- /dev/null +++ b/deployments/metrics/jaeger-deployment.yaml @@ -0,0 +1,82 @@ +# Copyright 2026 The kpt Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: jaeger + namespace: porch-monitoring # kpt-set: ${namespace} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger + namespace: porch-monitoring # kpt-set: ${namespace} +spec: + replicas: 1 + selector: + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + spec: + serviceAccountName: jaeger + containers: + - name: jaeger + image: docker.io/jaegertracing/all-in-one:latest # kpt-set: ${jaeger-image} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 4317 # kpt-set: ${jaeger-otlp-container-port} + name: otlp-grpc + - containerPort: 16686 # kpt-set: ${jaeger-ui-container-port} + name: ui + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1024Mi" + +--- +apiVersion: v1 +kind: Service +metadata: + name: jaeger-otlp + namespace: porch-monitoring # kpt-set: ${namespace} +spec: + ports: + - port: 4317 # kpt-set: ${jaeger-otlp-container-port} + protocol: TCP + targetPort: 4317 # kpt-set: ${jaeger-otlp-container-port} + name: otlp-grpc + selector: + app: jaeger + +--- +apiVersion: v1 +kind: Service +metadata: + name: jaeger-http + namespace: porch-monitoring # kpt-set: ${namespace} +spec: + ports: + - port: 16686 # kpt-set: ${jaeger-ui-container-port} + protocol: TCP + targetPort: 16686 # kpt-set: ${jaeger-ui-container-port} + name: ui + selector: + app: jaeger diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index 874630dc0..ab8693071 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -71,7 +71,11 @@ spec: - name: OTEL_EXPORTER_PROMETHEUS_PORT value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER - value: none + value: otlp + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: http://jaeger-otlp.porch-monitoring.svc.cluster.local:4317 + - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + value: grpc - name: OTEL_SERVICE_NAME value: porch-function-runner - name: PORCH_PPROF_PORT diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 94fc7307a..8a63016ad 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -93,7 +93,11 @@ spec: - name: OTEL_EXPORTER_PROMETHEUS_PORT value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER - value: none + value: otlp + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: http://jaeger-otlp.porch-monitoring.svc.cluster.local:4317 + - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + value: grpc - name: PORCH_PPROF_PORT value: "8080" args: diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index 573de63de..0a905cd6c 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -91,7 +91,11 @@ spec: - name: OTEL_EXPORTER_PROMETHEUS_PORT value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER - value: none + value: otlp + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: http://jaeger-otlp.porch-monitoring.svc.cluster.local:4317 + - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + value: grpc - name: PORCH_PPROF_PORT value: "8080" envFrom: diff --git a/scripts/deploy-monitoring.sh b/scripts/deploy-monitoring.sh index a1b5d9e54..ca60cf30b 100755 --- a/scripts/deploy-monitoring.sh +++ b/scripts/deploy-monitoring.sh @@ -33,6 +33,9 @@ GRAFANA_CONTAINER_PORT="${GRAFANA_CONTAINER_PORT:-3000}" GRAFANA_NODEPORT="${GRAFANA_NODEPORT:-30301}" PYROSCOPE_LOCAL_PORT="${PYROSCOPE_LOCAL_PORT:-4040}" PYROSCOPE_CONTAINER_PORT="${PYROSCOPE_CONTAINER_PORT:-4040}" +JAEGER_LOCAL_PORT="${JAEGER_LOCAL_PORT:-16686}" +JAEGER_OTLP_CONTAINER_PORT="${JAEGER_OTLP_CONTAINER_PORT:-4317}" +JAEGER_UI_CONTAINER_PORT="${JAEGER_UI_CONTAINER_PORT:-16686}" GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-porch}" GRAFANA_ADMIN_PW="${GRAFANA_ADMIN_PW:-}" @@ -46,6 +49,8 @@ GRAFANA_VERSION="${GRAFANA_VERSION:-latest}" GRAFANA_IMAGE="${DOCKERHUB_MIRROR}/grafana/grafana:${GRAFANA_VERSION}" PYROSCOPE_VERSION="${PYROSCOPE_VERSION:-latest}" PYROSCOPE_IMAGE="${DOCKERHUB_MIRROR}/grafana/pyroscope:${PYROSCOPE_VERSION}" +JAEGER_VERSION="${JAEGER_VERSION:-latest}" +JAEGER_IMAGE="${DOCKERHUB_MIRROR}/jaegertracing/all-in-one:${JAEGER_VERSION}" ALLOY_VERSION="${ALLOY_VERSION:-latest}" ALLOY_IMAGE="${DOCKERHUB_MIRROR}/grafana/alloy:${ALLOY_VERSION}" POSTGRES_EXPORTER_VERSION="${POSTGRES_EXPORTER_VERSION:-v0.16.0}" @@ -94,7 +99,7 @@ metadata: annotations: config.kubernetes.io/local-config: "true" info: - description: Prometheus and Grafana monitoring stack for Porch + description: Prometheus, Grafana, Jaeger, and profiling stack for Porch pipeline: mutators: - image: ${KRM_FN_REGISTRY_URL}/apply-setters:v0.2.0 @@ -102,10 +107,13 @@ pipeline: prometheus-image: "${PROMETHEUS_IMAGE}" grafana-image: "${GRAFANA_IMAGE}" pyroscope-image: "${PYROSCOPE_IMAGE}" + jaeger-image: "${JAEGER_IMAGE}" alloy-image: "${ALLOY_IMAGE}" postgres-exporter-image: "${POSTGRES_EXPORTER_IMAGE}" prometheus-container-port: "${PROMETHEUS_CONTAINER_PORT}" grafana-container-port: "${GRAFANA_CONTAINER_PORT}" + jaeger-otlp-container-port: "${JAEGER_OTLP_CONTAINER_PORT}" + jaeger-ui-container-port: "${JAEGER_UI_CONTAINER_PORT}" postgres-exporter-container-port: "${POSTGRES_EXPORTER_CONTAINER_PORT}" prometheus-nodeport: "${PROMETHEUS_NODEPORT}" grafana-nodeport: "${GRAFANA_NODEPORT}" @@ -209,16 +217,20 @@ get_service_urls() { GRAFANA_PF_PID=$! kubectl port-forward -n "${NAMESPACE}" deployment/pyroscope "${PYROSCOPE_LOCAL_PORT}":"${PYROSCOPE_CONTAINER_PORT}" > /dev/null 2>&1 & PYROSCOPE_PF_PID=$! + kubectl port-forward -n "${NAMESPACE}" service/jaeger-http "${JAEGER_LOCAL_PORT}":"${JAEGER_UI_CONTAINER_PORT}" > /dev/null 2>&1 & + JAEGER_PF_PID=$! sleep 2 echo "${PROMETHEUS_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-prometheus-pf.pid echo "${GRAFANA_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-grafana-pf.pid echo "${PYROSCOPE_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-pyroscope-pf.pid + echo "${JAEGER_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-jaeger-pf.pid PROMETHEUS_URL="http://localhost:${PROMETHEUS_LOCAL_PORT}" GRAFANA_URL="http://localhost:${GRAFANA_LOCAL_PORT}" PYROSCOPE_URL="http://localhost:${PYROSCOPE_LOCAL_PORT}" + JAEGER_URL="http://localhost:${JAEGER_LOCAL_PORT}" echo "" log_info "==========================================" @@ -232,6 +244,10 @@ get_service_urls() { log_info " Password: ${GRAFANA_ADMIN_PW}" log_info " stored in: kubectl -n porch-monitoring get secrets --selector app=grafana -o yaml" log_info " Pyroscope: ${PYROSCOPE_URL}" + log_info " Jaeger: ${JAEGER_URL}" + echo "" + log_info " - Porch components export traces to Jaeger OTLP (grpc) at:" + log_info " jaeger-otlp.${NAMESPACE}.svc.cluster.local:${JAEGER_OTLP_CONTAINER_PORT}" echo "" log_info " - Alloy discovers annotated pods in the cluster via profiles.grafana.com/*" log_info " - porch-server, porch-controllers, function-runner (port name: pprof)" @@ -257,11 +273,11 @@ cleanup() { if kubectl get namespace "$NAMESPACE" &> /dev/null; then log_info "Deleting resources in namespace $NAMESPACE..." - kubectl delete deployment prometheus grafana pyroscope alloy postgres-exporter -n "$NAMESPACE" --ignore-not-found=true - kubectl delete service prometheus grafana pyroscope postgres-exporter -n "$NAMESPACE" --ignore-not-found=true + kubectl delete deployment prometheus grafana pyroscope alloy postgres-exporter jaeger -n "$NAMESPACE" --ignore-not-found=true + kubectl delete service prometheus grafana pyroscope postgres-exporter jaeger-otlp jaeger-http -n "$NAMESPACE" --ignore-not-found=true kubectl delete configmap prometheus-config alloy-config grafana-dashboards grafana-dashboards-provider grafana-datasources -n "$NAMESPACE" --ignore-not-found=true kubectl delete secret grafana-admin-creds postgres-exporter-db-creds -n "$NAMESPACE" --ignore-not-found=true - kubectl delete serviceaccount prometheus pyroscope alloy -n "$NAMESPACE" --ignore-not-found=true + kubectl delete serviceaccount prometheus pyroscope alloy jaeger -n "$NAMESPACE" --ignore-not-found=true kubectl delete clusterrole prometheus alloy --ignore-not-found=true kubectl delete clusterrolebinding prometheus alloy --ignore-not-found=true @@ -278,13 +294,14 @@ main() { local action="${1:-deploy}" case "$action" in deploy) - log_info "Starting deployment of Prometheus and Grafana..." + log_info "Starting deployment of monitoring stack..." check_kpt create_namespace deploy_monitoring wait_for_deployment prometheus wait_for_deployment grafana wait_for_deployment pyroscope + wait_for_deployment jaeger wait_for_deployment alloy wait_for_deployment postgres-exporter get_service_urls @@ -312,6 +329,8 @@ main() { echo " GRAFANA_VERSION - Grafana image version (default: latest)" echo " PYROSCOPE_VERSION - Pyroscope image version (default: latest)" echo " PYROSCOPE_LOCAL_PORT - Pyroscope local port-forward port (default: 4040)" + echo " JAEGER_VERSION - Jaeger image version (default: latest)" + echo " JAEGER_LOCAL_PORT - Jaeger UI local port-forward port (default: 16686)" echo " ALLOY_VERSION - Grafana Alloy image version (default: latest)" echo " POSTGRES_EXPORTER_VERSION - Postgres exporter image version (default: v0.16.0)" echo " POSTGRES_EXPORTER_CONTAINER_PORT - Postgres exporter port (default: 9187)" From 3604ca3c6978a028c1a9d9d918e9290924bae98b Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Sat, 18 Jul 2026 02:37:29 +0200 Subject: [PATCH 06/13] Some housekeeping tasks Signed-off-by: Rendre Greyling --- deployments/porch/2-function-runner.yaml | 6 +- deployments/porch/3-porch-server.yaml | 6 +- deployments/porch/9-controllers.yaml | 6 +- make/deploy.mk | 22 ++ scripts/deploy-monitoring.sh | 320 ++++++++++++++++----- test/performance/README.md | 345 ++++++++++++++++------- 6 files changed, 505 insertions(+), 200 deletions(-) diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index ab8693071..874630dc0 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -71,11 +71,7 @@ spec: - name: OTEL_EXPORTER_PROMETHEUS_PORT value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER - value: otlp - - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - value: http://jaeger-otlp.porch-monitoring.svc.cluster.local:4317 - - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL - value: grpc + value: none - name: OTEL_SERVICE_NAME value: porch-function-runner - name: PORCH_PPROF_PORT diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 8a63016ad..94fc7307a 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -93,11 +93,7 @@ spec: - name: OTEL_EXPORTER_PROMETHEUS_PORT value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER - value: otlp - - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - value: http://jaeger-otlp.porch-monitoring.svc.cluster.local:4317 - - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL - value: grpc + value: none - name: PORCH_PPROF_PORT value: "8080" args: diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index 0a905cd6c..573de63de 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -91,11 +91,7 @@ spec: - name: OTEL_EXPORTER_PROMETHEUS_PORT value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER - value: otlp - - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - value: http://jaeger-otlp.porch-monitoring.svc.cluster.local:4317 - - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL - value: grpc + value: none - name: PORCH_PPROF_PORT value: "8080" envFrom: diff --git a/make/deploy.mk b/make/deploy.mk index c7697ff89..b7ddf1956 100644 --- a/make/deploy.mk +++ b/make/deploy.mk @@ -176,3 +176,25 @@ setup-dev-env: PORCH_TEST_CLUSTER=porch-test setup-dev-env: GIT_REPO_NAME=porch-test setup-dev-env: ## Setup gitea, Metallb and test repository in kind cluster ./scripts/setup-dev-env.sh + +##@ Monitoring + +.PHONY: deploy-monitoring +deploy-monitoring:## Deploy Prometheus, Grafana, and Postgres Exporter + ./scripts/deploy-monitoring.sh deploy + +.PHONY: deploy-monitoring-jaeger +deploy-monitoring-jaeger:## Deploy Jaeger and enable trace export from porch components + ./scripts/deploy-monitoring.sh jaeger + +.PHONY: deploy-monitoring-pyroscope +deploy-monitoring-pyroscope:## Deploy Pyroscope and Alloy profiling stack + ./scripts/deploy-monitoring.sh pyroscope + +.PHONY: cleanup-monitoring +cleanup-monitoring:## Remove monitoring stack and disable porch trace export + ./scripts/deploy-monitoring.sh cleanup + +.PHONY: restart-monitoring +restart-monitoring:## Restart the base monitoring stack + ./scripts/deploy-monitoring.sh restart diff --git a/scripts/deploy-monitoring.sh b/scripts/deploy-monitoring.sh index ca60cf30b..bc525c316 100755 --- a/scripts/deploy-monitoring.sh +++ b/scripts/deploy-monitoring.sh @@ -25,6 +25,12 @@ fi # Configuration NAMESPACE="${NAMESPACE:-porch-monitoring}" +PORCH_NAMESPACE="${PORCH_NAMESPACE:-porch-system}" +PORCH_TRACE_DEPLOYMENTS=( + porch-server + function-runner + porch-controllers +) PROMETHEUS_LOCAL_PORT="${PROMETHEUS_LOCAL_PORT:-9092}" PROMETHEUS_CONTAINER_PORT="${PROMETHEUS_CONTAINER_PORT:-9090}" PROMETHEUS_NODEPORT="${PROMETHEUS_NODEPORT:-30091}" @@ -60,6 +66,20 @@ POSTGRES_DB_USER="${POSTGRES_DB_USER:-porch}" POSTGRES_DB_PASSWORD="${POSTGRES_DB_PASSWORD:-porch}" POSTGRES_EXPORTER_DATA_SOURCE_URI="${POSTGRES_EXPORTER_DATA_SOURCE_URI:-porch-postgresql.porch-system.svc.cluster.local:5432/porch?sslmode=disable}" +BASE_MANIFESTS=( + prometheus-deployment.yaml + grafana-deployment.yaml + postgres-exporter-deployment.yaml +) +JAEGER_MANIFESTS=( + jaeger-deployment.yaml +) +PYROSCOPE_MANIFESTS=( + pyroscope-deployment.yaml +) + +JAEGER_OTLP_ENDPOINT="${JAEGER_OTLP_ENDPOINT:-http://jaeger-otlp.${NAMESPACE}.svc.cluster.local:${JAEGER_OTLP_CONTAINER_PORT}}" + RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' @@ -87,9 +107,32 @@ check_kpt() { fi } +is_base_deployed() { + local deployment + for deployment in prometheus grafana postgres-exporter; do + if ! kubectl get deployment "$deployment" -n "$NAMESPACE" &> /dev/null; then + return 1 + fi + done + return 0 +} + +is_jaeger_deployed() { + kubectl get deployment jaeger -n "$NAMESPACE" &> /dev/null +} + +is_pyroscope_deployed() { + kubectl get deployment pyroscope -n "$NAMESPACE" &> /dev/null +} + prepare_manifests() { - local temp_dir=$(mktemp -d) - cp -r "${METRICS_DIR}"/* "$temp_dir/" + local temp_dir + temp_dir=$(mktemp -d) + local manifest + + for manifest in "$@"; do + cp "${METRICS_DIR}/${manifest}" "$temp_dir/" + done cat > "$temp_dir/Kptfile" < /dev/null; then + log_info "Enabling trace export on ${PORCH_NAMESPACE}/${deployment}..." + kubectl set env deployment/"$deployment" -n "$PORCH_NAMESPACE" "${trace_env[@]}" + else + log_info "Skipping trace export for ${PORCH_NAMESPACE}/${deployment} (not deployed)" + fi + done +} + +disable_porch_trace_export() { + local deployment + local -a trace_env=( + "OTEL_TRACES_EXPORTER=none" + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT-" + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL-" + ) + + for deployment in "${PORCH_TRACE_DEPLOYMENTS[@]}"; do + if kubectl get deployment "$deployment" -n "$PORCH_NAMESPACE" &> /dev/null; then + log_info "Disabling trace export on ${PORCH_NAMESPACE}/${deployment}..." + kubectl set env deployment/"$deployment" -n "$PORCH_NAMESPACE" "${trace_env[@]}" + fi + done +} + stop_port_forwards() { if find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' -exec pkill -F '{}' \; 2>/dev/null; then find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' ! -wholename "${PORT_FORWARD_DIR}*" -exec rm '{}' \; 2>/dev/null || true @@ -211,26 +363,24 @@ get_service_urls() { stop_port_forwards sleep 2 - kubectl port-forward -n "${NAMESPACE}" deployment/prometheus "${PROMETHEUS_LOCAL_PORT}":"${PROMETHEUS_CONTAINER_PORT}" > /dev/null 2>&1 & - PROMETHEUS_PF_PID=$! - kubectl port-forward -n "${NAMESPACE}" deployment/grafana "${GRAFANA_LOCAL_PORT}":"${GRAFANA_CONTAINER_PORT}" > /dev/null 2>&1 & - GRAFANA_PF_PID=$! - kubectl port-forward -n "${NAMESPACE}" deployment/pyroscope "${PYROSCOPE_LOCAL_PORT}":"${PYROSCOPE_CONTAINER_PORT}" > /dev/null 2>&1 & - PYROSCOPE_PF_PID=$! - kubectl port-forward -n "${NAMESPACE}" service/jaeger-http "${JAEGER_LOCAL_PORT}":"${JAEGER_UI_CONTAINER_PORT}" > /dev/null 2>&1 & - JAEGER_PF_PID=$! + if is_base_deployed; then + kubectl port-forward -n "${NAMESPACE}" deployment/prometheus "${PROMETHEUS_LOCAL_PORT}":"${PROMETHEUS_CONTAINER_PORT}" > /dev/null 2>&1 & + echo "$!" > "${PORT_FORWARD_DIR}"/porch-prometheus-pf.pid + kubectl port-forward -n "${NAMESPACE}" deployment/grafana "${GRAFANA_LOCAL_PORT}":"${GRAFANA_CONTAINER_PORT}" > /dev/null 2>&1 & + echo "$!" > "${PORT_FORWARD_DIR}"/porch-grafana-pf.pid + fi - sleep 2 + if is_pyroscope_deployed; then + kubectl port-forward -n "${NAMESPACE}" deployment/pyroscope "${PYROSCOPE_LOCAL_PORT}":"${PYROSCOPE_CONTAINER_PORT}" > /dev/null 2>&1 & + echo "$!" > "${PORT_FORWARD_DIR}"/porch-pyroscope-pf.pid + fi - echo "${PROMETHEUS_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-prometheus-pf.pid - echo "${GRAFANA_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-grafana-pf.pid - echo "${PYROSCOPE_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-pyroscope-pf.pid - echo "${JAEGER_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-jaeger-pf.pid + if is_jaeger_deployed; then + kubectl port-forward -n "${NAMESPACE}" service/jaeger-http "${JAEGER_LOCAL_PORT}":"${JAEGER_UI_CONTAINER_PORT}" > /dev/null 2>&1 & + echo "$!" > "${PORT_FORWARD_DIR}"/porch-jaeger-pf.pid + fi - PROMETHEUS_URL="http://localhost:${PROMETHEUS_LOCAL_PORT}" - GRAFANA_URL="http://localhost:${GRAFANA_LOCAL_PORT}" - PYROSCOPE_URL="http://localhost:${PYROSCOPE_LOCAL_PORT}" - JAEGER_URL="http://localhost:${JAEGER_LOCAL_PORT}" + sleep 2 echo "" log_info "==========================================" @@ -238,27 +388,36 @@ get_service_urls() { log_info "==========================================" echo "" log_info "Access via port-forward (recommended):" - log_info " Prometheus: ${PROMETHEUS_URL}" - log_info " Grafana: ${GRAFANA_URL}" - log_info " Username: ${GRAFANA_ADMIN_USER}" - log_info " Password: ${GRAFANA_ADMIN_PW}" - log_info " stored in: kubectl -n porch-monitoring get secrets --selector app=grafana -o yaml" - log_info " Pyroscope: ${PYROSCOPE_URL}" - log_info " Jaeger: ${JAEGER_URL}" - echo "" - log_info " - Porch components export traces to Jaeger OTLP (grpc) at:" - log_info " jaeger-otlp.${NAMESPACE}.svc.cluster.local:${JAEGER_OTLP_CONTAINER_PORT}" - echo "" - log_info " - Alloy discovers annotated pods in the cluster via profiles.grafana.com/*" - log_info " - porch-server, porch-controllers, function-runner (port name: pprof)" - log_info "" - log_info " - Prometheus is scraping metrics from:" - log_info " - porch-server port 9464" - log_info " - porch-controller port 9464" - log_info " - function-runner port 9464" - log_info " - postgres-exporter port ${POSTGRES_EXPORTER_CONTAINER_PORT}" - log_info " - perf tests on host 172.17.0.1:9095 (when -enable-prometheus is set)" - log_info "" + + if is_base_deployed; then + log_info " Prometheus: http://localhost:${PROMETHEUS_LOCAL_PORT}" + log_info " Grafana: http://localhost:${GRAFANA_LOCAL_PORT}" + log_info " Username: ${GRAFANA_ADMIN_USER}" + log_info " Password: ${GRAFANA_ADMIN_PW}" + log_info " stored in: kubectl -n ${NAMESPACE} get secrets --selector app=grafana -o yaml" + echo "" + log_info " - Prometheus is scraping metrics from:" + log_info " - porch-server port 9464" + log_info " - porch-controller port 9464" + log_info " - function-runner port 9464" + log_info " - postgres-exporter port ${POSTGRES_EXPORTER_CONTAINER_PORT}" + log_info " - perf tests on host 172.17.0.1:9095 (when -enable-prometheus is set)" + fi + + if is_pyroscope_deployed; then + log_info " Pyroscope: http://localhost:${PYROSCOPE_LOCAL_PORT}" + echo "" + log_info " - Alloy discovers annotated pods in the cluster via profiles.grafana.com/*" + log_info " - porch-server, porch-controllers, function-runner (port name: pprof)" + fi + + if is_jaeger_deployed; then + log_info " Jaeger: http://localhost:${JAEGER_LOCAL_PORT}" + echo "" + log_info " - Porch components export traces to Jaeger OTLP (grpc) at:" + log_info " jaeger-otlp.${NAMESPACE}.svc.cluster.local:${JAEGER_OTLP_CONTAINER_PORT}" + fi + echo "" log_info "To stop port forwarding, run:" log_info " find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' -exec pkill -F '{}' \;" @@ -270,6 +429,7 @@ cleanup() { log_info "Stopping port forwarding..." stop_port_forwards + disable_porch_trace_export if kubectl get namespace "$NAMESPACE" &> /dev/null; then log_info "Deleting resources in namespace $NAMESPACE..." @@ -294,17 +454,16 @@ main() { local action="${1:-deploy}" case "$action" in deploy) - log_info "Starting deployment of monitoring stack..." - check_kpt - create_namespace - deploy_monitoring - wait_for_deployment prometheus - wait_for_deployment grafana - wait_for_deployment pyroscope - wait_for_deployment jaeger - wait_for_deployment alloy - wait_for_deployment postgres-exporter - get_service_urls + log_info "Starting deployment of base monitoring stack..." + deploy_base + ;; + jaeger) + log_info "Starting deployment of Jaeger..." + deploy_jaeger + ;; + pyroscope) + log_info "Starting deployment of Pyroscope..." + deploy_pyroscope ;; cleanup) check_kpt @@ -314,14 +473,23 @@ main() { check_kpt cleanup sleep 2 - main deploy + deploy_base ;; *) log_error "Unknown action: $action" - echo "Usage: $0 {deploy|cleanup|restart}" + echo "Usage: $0 {deploy|jaeger|pyroscope|cleanup|restart}" + echo "" + echo "Actions:" + echo " deploy Deploy Prometheus, Grafana, and Postgres Exporter (default)" + echo " jaeger Deploy Jaeger (deploys base stack first if needed)" + echo " pyroscope Deploy Pyroscope and Alloy (deploys base stack first if needed)" + echo " cleanup Remove all monitoring resources" + echo " restart Cleanup and redeploy the base stack" echo "" echo "Environment variables:" echo " NAMESPACE - Kubernetes namespace (default: porch-monitoring)" + echo " PORCH_NAMESPACE - Porch workloads namespace (default: porch-system)" + echo " JAEGER_OTLP_ENDPOINT - Jaeger OTLP endpoint for porch trace export" echo " PROMETHEUS_NODEPORT - Prometheus NodePort (default: 30091)" echo " GRAFANA_NODEPORT - Grafana NodePort (default: 30301)" echo " DOCKERHUB_MIRROR - Registry mirror for images (default: docker.io)" diff --git a/test/performance/README.md b/test/performance/README.md index 8feb617da..168b8a680 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -1,160 +1,287 @@ -Copyright 2024 The Nephio Authors +# Porch Performance Testing -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -# Porch Performance Testing Setup Guide +Performance tests measure Porch package revision lifecycle latency and throughput under configurable load. They create Gitea and Porch repositories, run package revisions through the full lifecycle (create → update resources → propose → approve/publish), optionally delete them, and record per-operation timings plus optional Prometheus metrics. ## Prerequisites + - Docker -- Kubernetes CLI (kubectl) -- Kpt CLI +- Kubernetes CLI (`kubectl`) +- [kpt CLI](https://kpt.dev/installation/) - Go development environment -- Access to GitHub repositories +- A running Porch deployment with Gitea (see setup below) ## 1. Set Up Development Environment -Run the following script from the porch directory to set up the development environment: + +From the repository root, set up the Kind cluster, Gitea, and test repository: + +```bash +make setup-dev-env +``` + +Or run the script directly: + ```bash ./scripts/setup-dev-env.sh ``` ## 2. Build and Deploy Porch -Execute the command below to build and deploy Porch in a Kubernetes Kind cluster: + +Build and deploy Porch into a Kind cluster. Choose the target that matches the API version and cache backend you want to test: + +| Make target | Cache | v1alpha2 | Notes | +|---|---|---|---| +| `make run-in-kind` | CR | no | Default; uses v1alpha1 PackageRevision API | +| `make run-in-kind-db-cache` | DB (PostgreSQL) | no | Database-backed cache | +| `make run-in-kind-v1alpha2` | DB | yes | Required for `-api-version=v1alpha2` tests | +| `make run-in-kind-v1alpha2-no-controller` | DB | yes | Exposes function-runner; run controller locally | + ```bash +# v1alpha1 (default) make run-in-kind + +# v1alpha2 with DB cache +make run-in-kind-v1alpha2 +``` + +Verify pods are running in `porch-system` before starting tests. + +To tear down: + +```bash +make destroy ``` -Once deployment is complete, you should see the related pods running. ## 3. Deploy Monitoring Stack (Optional) +Deploy the monitoring stack before running tests with `-enable-prometheus=true`. The test process exposes OTel metrics on **port 9095** on the host; Prometheus inside the Kind cluster scrapes them via the Docker gateway IP (`172.17.0.1:9095`). + +### Base stack (Prometheus, Grafana, Postgres Exporter) + ```bash +make deploy-monitoring +# or ./scripts/deploy-monitoring.sh deploy ``` -This will: -- Create a `porch-monitoring` namespace -- Deploy Prometheus on port 9090 inside the cluster -- Deploy Grafana on port 3000 inside the cluster -- Configure Prometheus as a Grafana datasource -- Load the Porch performance dashboard -- Setup port-forwarding: Prometheus on localhost:9092, Grafana on localhost:3001 +This creates the `porch-monitoring` namespace and deploys: + +- **Prometheus** — scrapes porch-server, porch-controllers, function-runner (port 9464), postgres-exporter, and perf-test metrics (host `172.17.0.1:9095`) +- **Grafana** — pre-loaded with the Porch performance dashboard +- **Postgres Exporter** — PostgreSQL metrics from `porch-postgresql` + +Port-forwarding is started automatically: -**Important for kind clusters**: Prometheus is configured to scrape metrics from the host machine via the Docker gateway IP (172.17.0.1). The performance test exposes metrics on port 9095 on the host, and Prometheus running inside the kind cluster accesses them via `172.17.0.1:9095`. +| Service | URL | +|---|---| +| Prometheus | http://localhost:9092 | +| Grafana | http://localhost:3001 | -### Access the Monitoring Tools +Grafana credentials are printed on deploy (default user: `porch`). They are also stored in the `grafana-admin-creds` secret in `porch-monitoring`. -- **Prometheus**: http://localhost:9092 (via port-forward) -- **Grafana**: http://localhost:3001/dashboards +### Optional: Jaeger (distributed tracing) + +Deploys Jaeger and enables OTLP trace export on porch-server, function-runner, and porch-controllers: -### Cleanup Monitoring Stack ```bash -./deploy-monitoring.sh cleanup +make deploy-monitoring-jaeger +# or +./scripts/deploy-monitoring.sh jaeger ``` -This will delete all monitoring resources and stop port forwarding. -### Restart Monitoring Stack +Jaeger UI: http://localhost:16686 + +### Optional: Pyroscope (continuous profiling) + +Deploys Pyroscope and Grafana Alloy, which discover annotated Porch pods via `profiles.grafana.com/*` annotations: + ```bash -./deploy-monitoring.sh restart +make deploy-monitoring-pyroscope +# or +./scripts/deploy-monitoring.sh pyroscope ``` -## 4. Run Performance Tests -Navigate to the performance test directory: +Pyroscope UI: http://localhost:4040 + +### Cleanup and restart + +```bash +make cleanup-monitoring # remove all monitoring resources +make restart-monitoring # cleanup + redeploy base stack +# or +./scripts/deploy-monitoring.sh cleanup +./scripts/deploy-monitoring.sh restart +``` + +**Important:** Only enable `-enable-prometheus=true` when the monitoring stack is deployed. The test waits 15 seconds before shutting down its metrics server so Prometheus can scrape final values. + +## 4. Test Packages + +Three packages are available under `packages/`. Select one with `-package-path`: + +| Package | Path | Description | +|---|---|---| +| **Small** (default) | `packages/small-package` | Single Deployment; `set-namespace` + `apply-setters` | +| **Complex** | `packages/complex-package` | 10 KRM resource files; 15 kpt catalog functions (mutators + validators) | +| **Large** | `packages/large-package` | 30 microservice Deployments in a single ~550 KB manifest; 15 kpt catalog functions | + +### Large package requirements + +The large package generates significantly larger request payloads than the default 6 MB `--max-request-body-size` (set on both porch-server and function-runner). **Only use `packages/large-package` when:** + +1. **Porch server, function-runner, and porch-controllers have adequate CPU and memory requests/limits** for rendering and reconciling large packages with a full kpt pipeline (15 functions across 30 Deployments). +2. **`--max-request-body-size` is increased significantly** on both porch-server and function-runner (they must stay in sync). The default is `6291456` (6 MB) in `deployments/porch/3-porch-server.yaml` and `deployments/porch/2-function-runner.yaml`. + +For reference, the default Kind deployment resource profiles are: + +| Component | CPU request | CPU limit | Memory request | Memory limit | +|---|---|---|---|---| +| porch-server | 250m | — | 256Mi | 2Gi | +| function-runner | 125m | — | 64Mi | — | +| porch-controllers | 500m | 1000m | 512Mi | 1Gi | + +These defaults are suitable for small and complex packages. Large-package workloads typically need higher limits across all three components. + +## 5. Run Performance Tests + +Tests are gated by environment variables and skipped otherwise. Run from the repository root or from `test/performance/`: + ```bash cd test/performance/ ``` -There are two main performance tests available: +### Scale / load test (`TestPorchScalePerformance`) + +Creates a configured number of repositories, packages, and revisions to simulate load. Runs repositories and packages in parallel up to the configured parallelism limits. -### Run Load Test -Creates a preset amount of repositories, packages, and revisions to simulate load on the Porch system. ```bash LOAD_TEST=1 go test -v ./... -timeout 1h ``` -Make sure to scale the timeout as needed and adjust test parameters as required. -Namely `-repos`, `-packages`, and `-revisions` to simulate the desired load and `-enable-prometheus` to enable visual metrics. -### Maximum Package Revisions Test -Tests the maximum number of package revisions that can be handled by Porch in a single repository. +Example with custom parameters: + ```bash -MAX_PR_TEST=1 go test -v ./... -timeout 1h +LOAD_TEST=1 go test -v ./... \ + -namespace=porch-metrics \ + -api-version=v1alpha1 \ + -repos=2 \ + -packages=3 \ + -revisions=5 \ + -repo-parallelism=2 \ + -package-parallelism=2 \ + -package-path=packages/complex-package \ + -enable-prometheus=true \ + -enable-deletion=true \ + -timeout 2h ``` -Make sure to scale the timeout as needed and adjust test parameters as required -> recommended is 72h. - -### Available Test Parameters: -- `-repos`: Number of repositories to test -- `-packages`: Number of packages per repository -- `-revisions`: Number of package revisions per package -- `-repo-parallelism`: Number of repositories to create in parallel -- `-package-parallelism`: Number of packages to create in parallel per repository -- `-enable-deletion`: Enable deletion of package revisions at the end of the test -- `-error-rate`: Maximum percentage of package revisions allowed to fail lifecycle transition -- `-enable-prometheus`: Enable Prometheus metrics server (default: false) -> **do not enable if monitoring stack is not deployed** -- `-metrics-log-prefix`: Prefix for the timestamped metrics log file -- `-results-file`: File name for test results -- `-detailed-log-file`: File name for detailed log -- `-repo-results-csv`: File name for repository results CSV -- `-operations-csv`: File name for operations details CSV -- `-deletion-csv`: File name for deletion operations CSV -- `-kptfile-path`: Path to the Kptfile -- `-package-resources-path`: Path to the package resources - -All results are stored in logs files located in the `test/performance/` and `test/performance/logs` directories. - -## 5. Sample Output + +### Maximum package revisions test (`TestIncreasePRsPerformance`) + +Creates package revisions sequentially in a single repository until the error-rate threshold is exceeded. Designed for long-running soak tests. ```bash -LOAD_TEST=1 go test -v ./test/performance -namespace=db-demo -repos=1 -packages=1 -revisions=3 -enable-prometheus=true -enable-deletion=true -timeout 1h +MAX_PR_TEST=1 go test -v ./... -timeout 72h +``` + +This test uses `-error-rate` (default `0.1`, i.e. 0.1%) to decide when to stop: once the fraction of failed revisions reaches the threshold, the test ends. Use a long timeout (72h or more is recommended). + +### API versions + +| Version | Deploy target | Lifecycle driver | +|---|---|---| +| `v1alpha1` (default) | `make run-in-kind` or `make run-in-kind-db-cache` | Direct lifecycle transitions via API | +| `v1alpha2` | `make run-in-kind-v1alpha2` | Controller-reconciled lifecycle with wait steps | + +v1alpha2 additionally records **Wait Ready**, **Wait Rendered**, and **Wait Published** operation timings while the controller reconciles each revision. + +Select the API version with `-api-version=v1alpha1` or `-api-version=v1alpha2`. + +### What each test does + +For every repository (`{namespace}-test-{N}`): + +1. Create a Gitea repository +2. Create a Porch `Repository` CR and wait for Ready +3. For each package (`network-function-{N}`) and revision: + - List existing package revisions + - Create a new package revision (init on v1, copy-from-published on subsequent revisions) + - Update package revision resources from the selected test package + - Propose → approve/publish (v1alpha2 waits for controller reconciliation between steps) +4. Optionally delete all created package revisions (`-enable-deletion=true`) + +Tests handle `SIGINT`/`SIGTERM` gracefully: in-flight work stops and results collected so far are written. + +### Test parameters + +| Flag | Default | Description | +|---|---|---| +| `-namespace` | `porch-metrics` | Kubernetes namespace for test resources | +| `-api-version` | `v1alpha1` | Porch API version (`v1alpha1` or `v1alpha2`) | +| `-repos` | `1` | Number of repositories | +| `-packages` | `1` | Packages per repository | +| `-revisions` | `1` | Revisions per package (ignored by MAX_PR_TEST, which runs until error threshold) | +| `-repo-parallelism` | `1` | Repositories created in parallel | +| `-package-parallelism` | `1` | Packages created in parallel per repository | +| `-package-path` | `packages/small-package` | Path to package resources directory | +| `-padding-size` | `0` | Pad package resources with N MB of data (0 = disabled) | +| `-error-rate` | `0.1` | Max failure percentage before MAX_PR_TEST stops (0.1 = 0.1%) | +| `-enable-deletion` | `false` | Delete all package revisions after lifecycle test | +| `-enable-prometheus` | `false` | Expose OTel metrics on host port 9095 | +| `-metrics-log-prefix` | `porch-metrics` | Prefix for timestamped log in `logs/` | +| `-results-file` | `load_test_results.txt` | Approved/deleted revision summary | +| `-detailed-log-file` | `load_test.log` | Per-operation detailed log | +| `-repo-results-csv` | `load_test_lifecycle_results.csv` | Per-revision lifecycle duration CSV | +| `-operations-csv` | `load_test_operations_results.csv` | Per-operation timing CSV | +| `-deletion-csv` | `load_test_deletion_results.csv` | Deletion operation CSV | +| `-gitea-url` | `http://localhost:3000` | Gitea API base URL | +| `-gitea-username` | `porch` | Gitea username | +| `-gitea-password` | `secret` | Gitea password | + +The KRM function registry URL is read from the `PORCH_GHCR_PREFIX_URL` environment variable (or `.env`), defaulting to `gcr.io/kptdev/krm-functions-catalog`. Package `Kptfile` placeholders `CHANGE_NAMESPACE` and `CHANGE_IMAGE` are substituted at runtime. + +## 6. Output Files + +Results are written relative to the working directory (typically `test/performance/`): + +| Output | Location | Contents | +|---|---|---| +| Timestamped test log | `logs/{prefix}-{api-version}-{timestamp}.log` | Consolidated results table | +| Results summary | `load_test_results.txt` (configurable) | Approved and deleted revision timings | +| Detailed log | `load_test.log` (configurable) | Per-operation log with configuration header | +| Lifecycle CSV | `load_test_lifecycle_results.csv` | Per package-revision total lifecycle duration | +| Operations CSV | `load_test_operations_results.csv` | Per-operation min/max/avg/total | +| Deletion CSV | `load_test_deletion_results.csv` | Deletion timings (when `-enable-deletion=true`) | + +## 7. Sample Output + +```bash +LOAD_TEST=1 go test -v ./... \ + -namespace=porch-metrics \ + -repos=1 -packages=1 -revisions=3 \ + -enable-prometheus=true \ + -enable-deletion=true \ + -timeout 1h ``` ``` -=== Consolidated Performance Test Results === +=== Consolidated Performance Test Results (v1alpha1) === Operation Min Max Avg Total ------------------------------------------------------------------------------------ -Create Gitea Repository R0 272ms 272ms 272ms 272ms -Create Porch Repository R0 3ms 3ms 3ms 3ms -Repository Ready Wait R0 2.004s 2.004s 2.004s 2.004s -Package Revision List v1 9ms 9ms 9ms 9ms -Package Revision List v2 6ms 6ms 6ms 6ms -Package Revision List v3 6ms 6ms 6ms 6ms -Package Revision Create v1 10ms 10ms 10ms 10ms -Package Revision Create v2 9ms 9ms 9ms 9ms -Package Revision Create v3 9ms 9ms 9ms 9ms -Package Revision Render v1 4ms 4ms 4ms 4ms -Package Revision Render v2 14ms 14ms 14ms 14ms -Package Revision Render v3 13ms 13ms 13ms 13ms -Package Revision Get Resources v1 3ms 3ms 3ms 3ms -Package Revision Get Resources v2 3ms 3ms 3ms 3ms -Package Revision Get Resources v3 3ms 3ms 3ms 3ms -Package Revision Update v1 10ms 10ms 10ms 10ms -Package Revision Update v2 10ms 10ms 10ms 10ms -Package Revision Update v3 9ms 9ms 9ms 9ms -Package Revision Render v1 14ms 14ms 14ms 14ms -Package Revision Render v2 10ms 10ms 10ms 10ms -Package Revision Render v3 10ms 10ms 10ms 10ms -Package Revision Get v1 2ms 2ms 2ms 2ms -Package Revision Get v2 2ms 2ms 2ms 2ms -Package Revision Get v3 2ms 2ms 2ms 2ms -Package Revision Propose v1 11ms 11ms 11ms 11ms -Package Revision Propose v2 9ms 9ms 9ms 9ms -Package Revision Propose v3 10ms 10ms 10ms 10ms -Package Revision Get (Proposed) v1 2ms 2ms 2ms 2ms -Package Revision Get (Proposed) v2 2ms 2ms 2ms 2ms -Package Revision Get (Proposed) v3 3ms 3ms 3ms 3ms -Package Revision Approve/Publish v1 348ms 348ms 348ms 348ms -Package Revision Approve/Publish v2 339ms 339ms 339ms 339ms -Package Revision Approve/Publish v3 340ms 340ms 340ms 340ms -Package Revision Propose Deletion v1 8ms 8ms 8ms 8ms -Package Revision Propose Deletion v2 6ms 6ms 6ms 6ms -Package Revision Propose Deletion v3 9ms 9ms 9ms 9ms -Package Revision Delete v1 262ms 262ms 262ms 262ms -Package Revision Delete v2 262ms 262ms 262ms 262ms -Package Revision Delete v3 269ms 269ms 269ms 269ms +Create Gitea Repository R0 272ms 272ms 272ms 272ms +Create Porch Repository R0 3ms 3ms 3ms 3ms +Repository Ready Wait R0 2.004s 2.004s 2.004s 2.004s +Package Revision List v1 9ms 9ms 9ms 9ms +Package Revision Create v1 10ms 10ms 10ms 10ms +Package Revision Get Resources v1 3ms 3ms 3ms 3ms +Package Revision Update v1 10ms 10ms 10ms 10ms +Package Revision Get v1 2ms 2ms 2ms 2ms +Package Revision Propose v1 11ms 11ms 11ms 11ms +Package Revision Get (Proposed) v1 2ms 2ms 2ms 2ms +Package Revision Approve/Publish v1 348ms 348ms 348ms 348ms +Package Revision Propose Deletion v1 8ms 8ms 8ms 8ms +Package Revision Delete v1 262ms 262ms 262ms 262ms +... +Total lifecycle duration for all operations: 12.5s +Tests completed! ``` + +With `-api-version=v1alpha2`, additional rows appear for **Wait Ready**, **Wait Rendered**, and **Wait Published** operations. From e6c43c384e05cd6d1f0b757f977fa5e94b52d594 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Mon, 20 Jul 2026 10:21:42 +0200 Subject: [PATCH 07/13] Address Copilot comments, add unit tests and some cleanup Signed-off-by: Rendre Greyling --- .env.template | 2 +- deployments/porch/2-function-runner.yaml | 16 - deployments/porch/3-porch-server.yaml | 16 - deployments/porch/9-controllers.yaml | 12 - internal/telemetry/metrics_test.go | 278 ++++++++++++++++++ internal/telemetry/pprof.go | 2 +- internal/telemetry/pprof_test.go | 121 ++++++++ internal/telemetry/pyroscope_test.go | 129 ++++++++ .../porch/packagerevision_approval.go | 4 +- scripts/deploy-monitoring.sh | 79 ++++- test/performance/logger.go | 2 +- 11 files changed, 610 insertions(+), 51 deletions(-) create mode 100644 internal/telemetry/pprof_test.go create mode 100644 internal/telemetry/pyroscope_test.go diff --git a/.env.template b/.env.template index 8c1abe5b3..1dedc4c00 100644 --- a/.env.template +++ b/.env.template @@ -1,4 +1,4 @@ # PORCH_GHCR_PREFIX_URL=/kptdev/krm-functions-catalog # DOCKERHUB_MIRROR= # DB_MAVEN_MIRROR= -# GRAFANA_ADMIN_PW +# GRAFANA_ADMIN_PW= diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index 874630dc0..2009c9ec7 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -31,18 +31,6 @@ spec: metadata: labels: app: function-runner - annotations: - profiles.grafana.com/service_name: porch-function-runner - profiles.grafana.com/cpu.scrape: "true" - profiles.grafana.com/cpu.port_name: "pprof" - profiles.grafana.com/memory.scrape: "true" - profiles.grafana.com/memory.port_name: "pprof" - profiles.grafana.com/goroutine.scrape: "true" - profiles.grafana.com/goroutine.port_name: "pprof" - profiles.grafana.com/block.scrape: "true" - profiles.grafana.com/block.port_name: "pprof" - profiles.grafana.com/mutex.scrape: "true" - profiles.grafana.com/mutex.port_name: "pprof" spec: serviceAccountName: porch-fn-runner containers: @@ -134,7 +122,3 @@ spec: protocol: TCP targetPort: 9464 name: metrics - - port: 8080 - protocol: TCP - targetPort: 8080 - name: pprof diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 94fc7307a..0a08a3a81 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -31,18 +31,6 @@ spec: metadata: labels: app: porch-server - annotations: - profiles.grafana.com/service_name: porch-server - profiles.grafana.com/cpu.scrape: "true" - profiles.grafana.com/cpu.port_name: "pprof" - profiles.grafana.com/memory.scrape: "true" - profiles.grafana.com/memory.port_name: "pprof" - profiles.grafana.com/goroutine.scrape: "true" - profiles.grafana.com/goroutine.port_name: "pprof" - profiles.grafana.com/block.scrape: "true" - profiles.grafana.com/block.port_name: "pprof" - profiles.grafana.com/mutex.scrape: "true" - profiles.grafana.com/mutex.port_name: "pprof" spec: serviceAccountName: porch-server volumes: @@ -160,9 +148,5 @@ spec: protocol: TCP targetPort: 9464 name: metrics - - port: 8080 - protocol: TCP - targetPort: 8080 - name: pprof selector: app: porch-server diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index 573de63de..327326af2 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -33,18 +33,6 @@ spec: metadata: labels: k8s-app: porch-controllers - annotations: - profiles.grafana.com/service_name: porch-controllers - profiles.grafana.com/cpu.scrape: "true" - profiles.grafana.com/cpu.port_name: "pprof" - profiles.grafana.com/memory.scrape: "true" - profiles.grafana.com/memory.port_name: "pprof" - profiles.grafana.com/goroutine.scrape: "true" - profiles.grafana.com/goroutine.port_name: "pprof" - profiles.grafana.com/block.scrape: "true" - profiles.grafana.com/block.port_name: "pprof" - profiles.grafana.com/mutex.scrape: "true" - profiles.grafana.com/mutex.port_name: "pprof" spec: serviceAccountName: porch-controllers securityContext: diff --git a/internal/telemetry/metrics_test.go b/internal/telemetry/metrics_test.go index 4b421538c..af6b2c3f9 100644 --- a/internal/telemetry/metrics_test.go +++ b/internal/telemetry/metrics_test.go @@ -16,6 +16,8 @@ package telemetry import ( "context" + "errors" + "flag" "testing" "time" @@ -25,6 +27,9 @@ import ( "go.opentelemetry.io/otel" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/klog/v2" ) // Remaining interface methods are not called by RecordPackageRevisionResourcesSize, @@ -123,3 +128,276 @@ func TestRecordPackageRevisionResourcesSize_RecordsMetrics(t *testing.T) { assert.True(t, foundHistogram, "expected porch_package_size_bytes histogram to be recorded") assert.True(t, foundGauge, "expected porch_package_size_bytes_total gauge to be recorded") } + +func setupMetricsTestMeterProvider(t *testing.T) *sdkmetric.ManualReader { + t.Helper() + + previousMp := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(mp) + t.Cleanup(func() { + otel.SetMeterProvider(previousMp) + mp.Shutdown(context.Background()) + }) + + require.NoError(t, InitMetrics()) + return reader +} + +func collectMetricData(t *testing.T, reader *sdkmetric.ManualReader) metricdata.ResourceMetrics { + t.Helper() + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + return rm +} + +func hasMetric(rm metricdata.ResourceMetrics, name string) bool { + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return true + } + } + } + return false +} + +func TestRecordAPICallDuration(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + RecordAPICallDuration(ResourcePackageRevision, "GET", APIVersionV1Alpha1, 0.25) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_api_call_duration_seconds")) +} + +func TestRecordAPICallDuration_NilInstrument(t *testing.T) { + setupMetricsTestMeterProvider(t) + + before := apiCallDurationSeconds + apiCallDurationSeconds = nil + t.Cleanup(func() { apiCallDurationSeconds = before }) + + assert.NotPanics(t, func() { + RecordAPICallDuration(ResourcePackageRevision, "GET", APIVersionV1Alpha1, 0.5) + }) +} + +func TestRecordRequestCount(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "test-user"}) + RecordRequestCount(ctx, ResourcePackageRevision, "LIST", APIVersionV1Alpha1) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_api_requests_by_user")) +} + +func TestRecordRequestCount_UnknownUser(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + RecordRequestCount(context.Background(), ResourcePackageRevision, "LIST", APIVersionV1Alpha1) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_api_requests_by_user")) +} + +func TestRecordRequestCount_NilInstrument(t *testing.T) { + setupMetricsTestMeterProvider(t) + + before := RequestsTotal + RequestsTotal = nil + t.Cleanup(func() { RequestsTotal = before }) + + assert.NotPanics(t, func() { + RecordRequestCount(context.Background(), ResourcePackageRevision, "LIST", APIVersionV1Alpha1) + }) +} + +func TestRecordControllerRequestCount(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + RecordControllerRequestCount(ResourcePackageRevision, "UPDATE", APIVersionV1Alpha2) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_api_requests_by_user")) +} + +func TestRecordControllerRequestCount_NilInstrument(t *testing.T) { + setupMetricsTestMeterProvider(t) + + before := RequestsTotal + RequestsTotal = nil + t.Cleanup(func() { RequestsTotal = before }) + + assert.NotPanics(t, func() { + RecordControllerRequestCount(ResourcePackageRevision, "UPDATE", APIVersionV1Alpha2) + }) +} + +func TestRecordExternalRepoOperation(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "git-user"}) + RecordExternalRepoOperation(ctx, "clone", time.Now().Add(-50*time.Millisecond)) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_api_call_duration_seconds")) + assert.True(t, hasMetric(rm, "porch_api_requests_by_user")) +} + +func TestRecordExternalRepoDuration_NilInstrument(t *testing.T) { + setupMetricsTestMeterProvider(t) + + before := apiCallDurationSeconds + apiCallDurationSeconds = nil + t.Cleanup(func() { apiCallDurationSeconds = before }) + + assert.NotPanics(t, func() { + recordExternalRepoDuration("fetch", 1.0) + }) +} + +func TestRecordExternalRepoRequestCount(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "git-user"}) + RecordExternalRepoRequestCount(ctx, "push") + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_api_requests_by_user")) +} + +func TestRecordExternalRepoRequestCount_NilInstrument(t *testing.T) { + setupMetricsTestMeterProvider(t) + + before := RequestsTotal + RequestsTotal = nil + t.Cleanup(func() { RequestsTotal = before }) + + ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "git-user"}) + assert.NotPanics(t, func() { + RecordExternalRepoRequestCount(ctx, "push") + }) +} + +func TestRecordPackageRevisionResourcesSize_WithPath(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + fake := repository.PackageRevisionKey{ + PkgKey: repository.PackageKey{ + RepoKey: repository.RepositoryKey{Namespace: "test-ns", Name: "repo"}, + Path: "configs", + Package: "my-pkg", + }, + WorkspaceName: "ws", + Revision: 1, + } + + RecordPackageRevisionResourcesSize(context.Background(), fake, 2048) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_package_size_bytes")) + assert.True(t, hasMetric(rm, "porch_package_size_bytes_total")) +} + +func TestRecordPackageRevisionResourcesSize_VerboseLogging(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + klog.InitFlags(nil) + require.NoError(t, flag.Set("v", "3")) + + fake := repository.PackageRevisionKey{ + PkgKey: repository.PackageKey{RepoKey: repository.RepositoryKey{Namespace: "test-ns"}}, + WorkspaceName: "ws", + Revision: 1, + } + + assert.NotPanics(t, func() { + RecordPackageRevisionResourcesSize(context.Background(), fake, 512) + }) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_package_size_bytes")) +} + +func TestPerfTestRecordMetric(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestRecordMetric("create", APIVersionV1Alpha2, "repo", "pkg", 100*time.Millisecond, nil) + PerfTestRecordMetric("delete", APIVersionV1Alpha2, "repo", "pkg", 50*time.Millisecond, errors.New("failed")) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_operation_duration_seconds")) + assert.True(t, hasMetric(rm, "porch_perf_operations_total")) +} + +func TestPerfTestRecordLifecycleTransition(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestRecordLifecycleTransition("Draft", "Published", APIVersionV1Alpha2, "repo", "pkg", 2*time.Second, nil) + PerfTestRecordLifecycleTransition("Published", "Draft", APIVersionV1Alpha2, "repo", "pkg", time.Second, errors.New("failed")) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_lifecycle_transition_duration_seconds")) +} + +func TestPerfTestRecordPackageRevision(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestRecordPackageRevision("create", nil) + PerfTestRecordPackageRevision("delete", errors.New("failed")) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_package_revisions_total")) +} + +func TestPerfTestSetTestRunInfo(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestSetTestRunInfo("perf-suite", "default", APIVersionV1Alpha2, time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_test_run_info")) +} + +func TestPerfTestRecordActiveOperation(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestRecordActiveOperation("create", APIVersionV1Alpha2, 1) + PerfTestRecordActiveOperation("create", APIVersionV1Alpha2, -1) + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_active_operations")) +} + +func TestPerfTestIncrementRepositoryCounter(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestIncrementRepositoryCounter() + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_repositories_created_total")) +} + +func TestPerfTestIncrementPackageCounter(t *testing.T) { + reader := setupMetricsTestMeterProvider(t) + + PerfTestIncrementPackageCounter() + + rm := collectMetricData(t, reader) + assert.True(t, hasMetric(rm, "porch_perf_packages_created_total")) +} + +func TestStatusLabel(t *testing.T) { + assert.Equal(t, "success", statusLabel(nil)) + assert.Equal(t, "error", statusLabel(errors.New("boom"))) +} + +func TestGetK8sUserName(t *testing.T) { + assert.Equal(t, "", getK8sUserName(context.Background())) + + ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "alice"}) + assert.Equal(t, "alice", getK8sUserName(ctx)) +} diff --git a/internal/telemetry/pprof.go b/internal/telemetry/pprof.go index 63331d7a3..854d0eac3 100644 --- a/internal/telemetry/pprof.go +++ b/internal/telemetry/pprof.go @@ -86,7 +86,7 @@ func (p *Profiling) Stop() { func (p *Profiling) serve() { klog.Infof("Starting profiling server on port :%d", p.port) if err := p.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - klog.Errorf("Error starting metrics server: %v", err) + klog.Errorf("Error starting profiling server: %v", err) } klog.Info("Profiling server stopped") } diff --git a/internal/telemetry/pprof_test.go b/internal/telemetry/pprof_test.go new file mode 100644 index 000000000..1ce7355aa --- /dev/null +++ b/internal/telemetry/pprof_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "fmt" + "io" + "net" + "net/http" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProfiling_Start_NoEnv(t *testing.T) { + t.Setenv(PProfPortEnvVar, "") + + var p Profiling + p.Start() + + assert.Nil(t, p.server) +} + +func TestProfiling_Start_InvalidPort(t *testing.T) { + t.Setenv(PProfPortEnvVar, "not-a-number") + + var p Profiling + p.Start() + + assert.Nil(t, p.server) +} + +func TestProfiling_Start_Stop(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := lis.Addr().(*net.TCPAddr).Port + require.NoError(t, lis.Close()) + + t.Setenv(PProfPortEnvVar, strconv.Itoa(port)) + + var p Profiling + p.Start() + require.NotNil(t, p.server) + t.Cleanup(func() { p.Stop() }) + + require.Eventually(t, func() bool { + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/debug/pprof/", port)) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, 5*time.Second, 10*time.Millisecond) + + endpoints := []string{ + "/debug/pprof/cmdline", + "/debug/pprof/symbol", + "/debug/pprof/heap", + "/debug/pprof/goroutine", + "/debug/pprof/threadcreate", + "/debug/pprof/block", + "/debug/pprof/mutex", + "/debug/pprof/allocs", + } + for _, endpoint := range endpoints { + t.Run(endpoint, func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", port, endpoint)) + require.NoError(t, err) + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + } + + p.Stop() +} + +func TestProfiling_Start_PortAlreadyInUse(t *testing.T) { + lis, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := lis.Addr().(*net.TCPAddr).Port + t.Cleanup(func() { lis.Close() }) + + t.Setenv(PProfPortEnvVar, strconv.Itoa(port)) + + var p Profiling + p.Start() + require.NotNil(t, p.server) + + // serve() logs an error when ListenAndServe fails on an occupied port. + time.Sleep(50 * time.Millisecond) + + p.Stop() +} + +func TestProfiling_Stop_ShutdownError(t *testing.T) { + var p Profiling + p.server = &http.Server{Addr: ":0"} + assert.NotPanics(t, func() { p.Stop() }) +} + +func TestProfiling_Stop_NoServer(t *testing.T) { + var p Profiling + assert.NotPanics(t, func() { p.Stop() }) +} diff --git a/internal/telemetry/pyroscope_test.go b/internal/telemetry/pyroscope_test.go new file mode 100644 index 000000000..0783b0a2c --- /dev/null +++ b/internal/telemetry/pyroscope_test.go @@ -0,0 +1,129 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPyroscopeProfiling_Start_NoServer(t *testing.T) { + t.Setenv(PyroscopeServerEnvVar, "") + + var p PyroscopeProfiling + p.Start() + + assert.Nil(t, p.stop) +} + +func TestPyroscopeProfiling_Start_InvalidServer(t *testing.T) { + t.Setenv(PyroscopeServerEnvVar, "://invalid-url") + + var p PyroscopeProfiling + p.Start() + + assert.Nil(t, p.stop) +} + +func TestPyroscopeProfiling_Start_Success_DefaultAppName(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + t.Setenv(PyroscopeServerEnvVar, ts.URL) + t.Setenv(PyroscopeAppNameEnvVar, "") + t.Setenv(PyroscopeAuthUserVar, "") + t.Setenv(PyroscopeAuthPassVar, "") + t.Setenv(PyroscopeLogsEnabledEnvVar, "") + + var p PyroscopeProfiling + p.Start() + require.NotNil(t, p.stop) + t.Cleanup(func() { p.Stop() }) +} + +func TestPyroscopeProfiling_Start_Success_CustomConfig(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + t.Setenv(PyroscopeServerEnvVar, ts.URL) + t.Setenv(PyroscopeAppNameEnvVar, "custom-app") + t.Setenv(PyroscopeAuthUserVar, "user") + t.Setenv(PyroscopeAuthPassVar, "pass") + t.Setenv(PyroscopeLogsEnabledEnvVar, "1") + + var p PyroscopeProfiling + p.Start() + require.NotNil(t, p.stop) + p.Stop() + assert.Nil(t, p.stop) +} + +func TestPyroscopeProfiling_Start_LogsEnabledTrue(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + t.Setenv(PyroscopeServerEnvVar, ts.URL) + t.Setenv(PyroscopeAppNameEnvVar, "logs-app") + t.Setenv(PyroscopeLogsEnabledEnvVar, "true") + + var p PyroscopeProfiling + p.Start() + require.NotNil(t, p.stop) + p.Stop() +} + +func TestPyroscopeProfiling_Stop_NoProfiler(t *testing.T) { + var p PyroscopeProfiling + assert.NotPanics(t, func() { p.Stop() }) +} + +func TestPyroscopeProfiling_Stop_WithError(t *testing.T) { + var p PyroscopeProfiling + p.stop = func() error { + return errors.New("stop failed") + } + + assert.NotPanics(t, func() { p.Stop() }) + assert.Nil(t, p.stop) +} + +func TestPyroscopeProfiling_Stop_Success(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + t.Setenv(PyroscopeServerEnvVar, ts.URL) + t.Setenv(PyroscopeAppNameEnvVar, fmt.Sprintf("stop-test-%d", time.Now().UnixNano())) + + var p PyroscopeProfiling + p.Start() + require.NotNil(t, p.stop) + assert.NotPanics(t, func() { p.Stop() }) + assert.Nil(t, p.stop) +} diff --git a/pkg/registry/porch/packagerevision_approval.go b/pkg/registry/porch/packagerevision_approval.go index a76046560..d0d9616a9 100644 --- a/pkg/registry/porch/packagerevision_approval.go +++ b/pkg/registry/porch/packagerevision_approval.go @@ -63,7 +63,7 @@ func (a *packageRevisionApproval) Get(ctx context.Context, name string, _ *metav telemetry.RecordAPICallDuration(praTelemetryName, "GET", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "GET", telemetry.APIVersionV1Alpha1) + telemetry.RecordRequestCount(ctx, praTelemetryName, "GET", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) @@ -86,7 +86,7 @@ func (a *packageRevisionApproval) Update(ctx context.Context, name string, objIn telemetry.RecordAPICallDuration(praTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1, time.Since(start).Seconds()) }() - telemetry.RecordRequestCount(ctx, prTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1) + telemetry.RecordRequestCount(ctx, praTelemetryName, "UPDATE", telemetry.APIVersionV1Alpha1) ctx = pctx.WithNewRequestIDAndPackageRevision(ctx, name) diff --git a/scripts/deploy-monitoring.sh b/scripts/deploy-monitoring.sh index bc525c316..f0ec18a7a 100755 --- a/scripts/deploy-monitoring.sh +++ b/scripts/deploy-monitoring.sh @@ -31,6 +31,11 @@ PORCH_TRACE_DEPLOYMENTS=( function-runner porch-controllers ) +PORCH_PPROF_DEPLOYMENTS=( + porch-server:porch-server + function-runner:porch-function-runner + porch-controllers:porch-controllers +) PROMETHEUS_LOCAL_PORT="${PROMETHEUS_LOCAL_PORT:-9092}" PROMETHEUS_CONTAINER_PORT="${PROMETHEUS_CONTAINER_PORT:-9090}" PROMETHEUS_NODEPORT="${PROMETHEUS_NODEPORT:-30091}" @@ -304,6 +309,7 @@ deploy_pyroscope() { wait_for_deployment pyroscope wait_for_deployment alloy + enable_porch_pprof_annotations log_info "Pyroscope and Alloy deployed successfully" get_service_urls @@ -349,6 +355,74 @@ disable_porch_trace_export() { done } +enable_porch_pprof_annotations() { + local entry deployment service_name + for entry in "${PORCH_PPROF_DEPLOYMENTS[@]}"; do + deployment="${entry%%:*}" + service_name="${entry#*:}" + if kubectl get deployment "$deployment" -n "$PORCH_NAMESPACE" &> /dev/null; then + log_info "Enabling pprof annotations on ${PORCH_NAMESPACE}/${deployment}..." + kubectl patch deployment "$deployment" -n "$PORCH_NAMESPACE" --type merge -p "$(cat < /dev/null; then + log_info "Disabling pprof annotations on ${PORCH_NAMESPACE}/${deployment}..." + kubectl patch deployment "$deployment" -n "$PORCH_NAMESPACE" --type merge -p '{ + "spec": { + "template": { + "metadata": { + "annotations": { + "profiles.grafana.com/service_name": null, + "profiles.grafana.com/cpu.scrape": null, + "profiles.grafana.com/cpu.port_name": null, + "profiles.grafana.com/memory.scrape": null, + "profiles.grafana.com/memory.port_name": null, + "profiles.grafana.com/goroutine.scrape": null, + "profiles.grafana.com/goroutine.port_name": null, + "profiles.grafana.com/block.scrape": null, + "profiles.grafana.com/block.port_name": null, + "profiles.grafana.com/mutex.scrape": null, + "profiles.grafana.com/mutex.port_name": null + } + } + } + } +}' + fi + done +} + stop_port_forwards() { if find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' -exec pkill -F '{}' \; 2>/dev/null; then find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' ! -wholename "${PORT_FORWARD_DIR}*" -exec rm '{}' \; 2>/dev/null || true @@ -407,8 +481,8 @@ get_service_urls() { if is_pyroscope_deployed; then log_info " Pyroscope: http://localhost:${PYROSCOPE_LOCAL_PORT}" echo "" - log_info " - Alloy discovers annotated pods in the cluster via profiles.grafana.com/*" - log_info " - porch-server, porch-controllers, function-runner (port name: pprof)" + log_info " - Alloy discovers porch-server, porch-controllers, and function-runner" + log_info " via profiles.grafana.com/* annotations (port name: pprof)" fi if is_jaeger_deployed; then @@ -430,6 +504,7 @@ cleanup() { log_info "Stopping port forwarding..." stop_port_forwards disable_porch_trace_export + disable_porch_pprof_annotations if kubectl get namespace "$NAMESPACE" &> /dev/null; then log_info "Deleting resources in namespace $NAMESPACE..." diff --git a/test/performance/logger.go b/test/performance/logger.go index bde0d4c22..e10a67053 100644 --- a/test/performance/logger.go +++ b/test/performance/logger.go @@ -25,7 +25,7 @@ import ( pkgerrors "github.com/pkg/errors" ) -const timeDateFormat = "2006-01-02 15:04:05" +const timeDateFormat = "2006-01-02-15-04-05" type TestLogger struct { file *os.File From 912a82e818a29d8ffff52cdcbce917a27112a5b2 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Wed, 29 Jul 2026 16:11:52 +0200 Subject: [PATCH 08/13] Cleanup on kptfiles and dashboards Signed-off-by: Rendre Greyling --- .../grafana-perf-test-dashboard.json | 51 ++++++++++++++++--- .../grafana-porch-server-dashboard.json | 29 ++++++++--- .../grafana-resource-usage-dashboard.json | 33 ++---------- deployments/metrics/jaeger-deployment.yaml | 2 +- test/performance/README.md | 3 +- test/performance/driver_v1alpha2.go | 2 +- .../packages/complex-package/Kptfile | 30 +++++------ .../packages/large-package/Kptfile | 30 +++++------ .../packages/small-package/Kptfile | 4 +- test/performance/performance_suite.go | 16 ------ 10 files changed, 102 insertions(+), 98 deletions(-) diff --git a/deployments/metrics-resources/grafana-perf-test-dashboard.json b/deployments/metrics-resources/grafana-perf-test-dashboard.json index 6fc7e0050..c5ae4d1f6 100644 --- a/deployments/metrics-resources/grafana-perf-test-dashboard.json +++ b/deployments/metrics-resources/grafana-perf-test-dashboard.json @@ -2755,24 +2755,59 @@ "templating": { "list": [ { - "name": "namespace", - "type": "query", - "query": "label_values(porch_perf_test_run_info, namespace)", + "allValue": ".*", "current": { + "selected": true, "text": "porch-metrics", "value": "porch-metrics" }, - "refresh": 1 + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(porch_perf_test_run_info, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(porch_perf_test_run_info, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" }, { - "name": "api_version", - "type": "query", - "query": "label_values(porch_perf_test_run_info, api_version)", "current": { + "selected": true, "text": "v1alpha1", "value": "v1alpha1" }, - "refresh": 1 + "hide": 0, + "includeAll": false, + "label": "API Version", + "multi": false, + "name": "api_version", + "options": [ + { + "selected": true, + "text": "v1alpha1", + "value": "v1alpha1" + }, + { + "selected": false, + "text": "v1alpha2", + "value": "v1alpha2" + } + ], + "query": "v1alpha1,v1alpha2", + "skipUrlSync": false, + "type": "custom" } ] }, diff --git a/deployments/metrics-resources/grafana-porch-server-dashboard.json b/deployments/metrics-resources/grafana-porch-server-dashboard.json index 9e45d1209..139ae542f 100644 --- a/deployments/metrics-resources/grafana-porch-server-dashboard.json +++ b/deployments/metrics-resources/grafana-porch-server-dashboard.json @@ -1351,18 +1351,31 @@ "templating": { "list": [ { - "name": "api_version", - "type": "query", - "query": "label_values(porch_api_call_duration_seconds, api_version)", "current": { + "selected": true, "text": "v1alpha1", "value": "v1alpha1" }, - "refresh": 1, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "hide": 0, + "includeAll": false, + "label": "API Version", + "multi": false, + "name": "api_version", + "options": [ + { + "selected": true, + "text": "v1alpha1", + "value": "v1alpha1" + }, + { + "selected": false, + "text": "v1alpha2", + "value": "v1alpha2" + } + ], + "query": "v1alpha1,v1alpha2", + "skipUrlSync": false, + "type": "custom" } ] }, diff --git a/deployments/metrics-resources/grafana-resource-usage-dashboard.json b/deployments/metrics-resources/grafana-resource-usage-dashboard.json index 142b58313..d4cb11a94 100644 --- a/deployments/metrics-resources/grafana-resource-usage-dashboard.json +++ b/deployments/metrics-resources/grafana-resource-usage-dashboard.json @@ -1065,17 +1065,8 @@ "uid": "prometheus" }, "expr": "process_resident_memory_bytes{job=\"porch-server\"}", - "legendFormat": "resident - {{instance}}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "process_virtual_memory_bytes{job=\"porch-server\"}", - "legendFormat": "virtual - {{instance}}", - "refId": "B" } ], "title": "Porch Server - Memory Usage", @@ -1145,17 +1136,8 @@ "uid": "prometheus" }, "expr": "process_resident_memory_bytes{job=\"function-runner\"}", - "legendFormat": "resident - {{instance}}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "process_virtual_memory_bytes{job=\"function-runner\"}", - "legendFormat": "virtual - {{instance}}", - "refId": "B" } ], "title": "Function Runner - Memory Usage", @@ -1225,17 +1207,8 @@ "uid": "prometheus" }, "expr": "process_resident_memory_bytes{job=\"porch-controllers\"}", - "legendFormat": "resident - {{instance}}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "process_virtual_memory_bytes{job=\"porch-controllers\"}", - "legendFormat": "virtual - {{instance}}", - "refId": "B" } ], "title": "Porch Controllers - Memory Usage", diff --git a/deployments/metrics/jaeger-deployment.yaml b/deployments/metrics/jaeger-deployment.yaml index 7b28f4f19..47f0c7d6a 100644 --- a/deployments/metrics/jaeger-deployment.yaml +++ b/deployments/metrics/jaeger-deployment.yaml @@ -49,7 +49,7 @@ spec: memory: "256Mi" cpu: "100m" limits: - memory: "1024Mi" + memory: "2Gi" --- apiVersion: v1 diff --git a/test/performance/README.md b/test/performance/README.md index 168b8a680..7cb93ceed 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -222,7 +222,6 @@ Tests handle `SIGINT`/`SIGTERM` gracefully: in-flight work stops and results col | `-repo-parallelism` | `1` | Repositories created in parallel | | `-package-parallelism` | `1` | Packages created in parallel per repository | | `-package-path` | `packages/small-package` | Path to package resources directory | -| `-padding-size` | `0` | Pad package resources with N MB of data (0 = disabled) | | `-error-rate` | `0.1` | Max failure percentage before MAX_PR_TEST stops (0.1 = 0.1%) | | `-enable-deletion` | `false` | Delete all package revisions after lifecycle test | | `-enable-prometheus` | `false` | Expose OTel metrics on host port 9095 | @@ -236,7 +235,7 @@ Tests handle `SIGINT`/`SIGTERM` gracefully: in-flight work stops and results col | `-gitea-username` | `porch` | Gitea username | | `-gitea-password` | `secret` | Gitea password | -The KRM function registry URL is read from the `PORCH_GHCR_PREFIX_URL` environment variable (or `.env`), defaulting to `gcr.io/kptdev/krm-functions-catalog`. Package `Kptfile` placeholders `CHANGE_NAMESPACE` and `CHANGE_IMAGE` are substituted at runtime. +The KRM function registry URL is configured via `PORCH_GHCR_PREFIX_URL` in the repo root `.env` file. It is applied to porch-server and function-runner at deploy time (`make run-in-kind` reads `.env` automatically). Package `Kptfile` images use short names (for example `set-namespace:v0.4.1`); function-runner resolves them with `--default-image-prefix`. The `CHANGE_NAMESPACE` placeholder in Kptfiles is substituted at test runtime. ## 6. Output Files diff --git a/test/performance/driver_v1alpha2.go b/test/performance/driver_v1alpha2.go index 1da168ea7..99174319a 100644 --- a/test/performance/driver_v1alpha2.go +++ b/test/performance/driver_v1alpha2.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const v1alpha2ControllerWaitTimeout = 120 * time.Second +const v1alpha2ControllerWaitTimeout = 300 * time.Second type v1alpha2Driver struct { baseDriver diff --git a/test/performance/packages/complex-package/Kptfile b/test/performance/packages/complex-package/Kptfile index 84d4f41ef..e77918bd6 100644 --- a/test/performance/packages/complex-package/Kptfile +++ b/test/performance/packages/complex-package/Kptfile @@ -9,11 +9,11 @@ info: pipeline: mutators: # 1. set-namespace — stamps the target namespace on every namespaced resource - - image: CHANGE_IMAGE/set-namespace:v0.4.1 + - image: set-namespace:v0.4.1 configMap: namespace: CHANGE_NAMESPACE # 2. create-setters — scans resources and inserts # kpt-set: ${name} setter comments - - image: CHANGE_IMAGE/create-setters:v0.1.0 + - image: create-setters:v0.1.0 configMap: app-name: complex-app environment: dev @@ -31,7 +31,7 @@ pipeline: team: platform version: v1.0.0 # 3. apply-setters — substitutes all # kpt-set: ${name} markers with configured values - - image: CHANGE_IMAGE/apply-setters:v0.2.0 + - image: apply-setters:v0.2.0 configMap: app-name: complex-app environment: dev @@ -49,7 +49,7 @@ pipeline: team: platform version: v1.0.0 # 4. set-labels — bulk-stamps labels onto every resource - - image: CHANGE_IMAGE/set-labels:v0.2.0 + - image: set-labels:v0.2.0 configMap: app: complex-app environment: dev @@ -57,25 +57,25 @@ pipeline: team: platform version: v1.0.0 # 5. set-annotations — bulk-stamps annotations onto every resource - - image: CHANGE_IMAGE/set-annotations:v0.1.4 + - image: set-annotations:v0.1.4 configMap: app.kubernetes.io/managed-by: kpt app.kubernetes.io/part-of: perf-test-suite team: platform # 6. set-image — updates the nginx container image name and tag - - image: CHANGE_IMAGE/set-image:v0.1.1 + - image: set-image:v0.1.1 configMap: name: nginx newName: docker.io/nginx newTag: latest # 7. search-replace — finds a specific field path and overwrites its value - - image: CHANGE_IMAGE/search-replace:v0.2.0 + - image: search-replace:v0.2.0 configMap: by-path: metadata.annotations.[app.kubernetes.io/managed-by] put-value: kpt # 8. ensure-name-substring — prepends 'complex-app-' to any resource whose # name does not already start with that prefix - - image: CHANGE_IMAGE/ensure-name-substring:v0.1.1 + - image: ensure-name-substring:v0.1.1 configMap: prepend: complex-app- selectors: @@ -92,26 +92,26 @@ pipeline: - kind: LimitRange # 9. apply-replacements — propagates field values across resources as defined # in the ApplyReplacements resource inside the package - - image: CHANGE_IMAGE/apply-replacements:v0.1.1 + - image: apply-replacements:v0.1.1 configPath: apply-replacements.yaml # 10. starlark — executes the StarlarkRun script from the package - - image: CHANGE_IMAGE/starlark:v0.4.3 + - image: starlark:v0.4.3 configPath: starlark-run.yaml # 11. upsert-resource — inserts or updates the LimitRange into the package - - image: CHANGE_IMAGE/upsert-resource:v0.2.0 + - image: upsert-resource:v0.2.0 configPath: limitrange.yaml # 12. generate-folders — generates Config Controller Folder resources; no-op # when no HierarchyConfig resources are present in the package - - image: CHANGE_IMAGE/generate-folders:v0.1.1 + - image: generate-folders:v0.1.1 # 13. set-enforcement-action — sets enforcementAction: warn on all Constraint # resources (defined in gatekeeper-constraints.yaml) - - image: CHANGE_IMAGE/set-enforcement-action:v0.1.1 + - image: set-enforcement-action:v0.1.1 configMap: enforcementAction: warn validators: # 14. kubeconform — validates KRM resources against their JSON schemas # (uses baked-in schemas; see https://catalog.kpt.dev/kubeconform/v0.1/) - - image: CHANGE_IMAGE/kubeconform:v0.1.1 + - image: kubeconform:v0.1.1 # 15. gatekeeper — validates resources against ConstraintTemplate/Constraint # policies defined in gatekeeper-constraints.yaml - - image: CHANGE_IMAGE/gatekeeper:v0.2.1 + - image: gatekeeper:v0.2.1 diff --git a/test/performance/packages/large-package/Kptfile b/test/performance/packages/large-package/Kptfile index eb8ee5577..3a1c1236c 100644 --- a/test/performance/packages/large-package/Kptfile +++ b/test/performance/packages/large-package/Kptfile @@ -9,11 +9,11 @@ info: pipeline: mutators: # 1. set-namespace — stamps the target namespace on every namespaced resource - - image: CHANGE_IMAGE/set-namespace:v0.4.1 + - image: set-namespace:v0.4.1 configMap: namespace: CHANGE_NAMESPACE # 2. create-setters — scans resources and inserts # kpt-set: ${name} setter comments - - image: CHANGE_IMAGE/create-setters:v0.1.0 + - image: create-setters:v0.1.0 configMap: app-name: large-app environment: dev @@ -43,7 +43,7 @@ pipeline: memory-limit: 256Mi image-pull-secret: registry-credentials # 3. apply-setters — substitutes all # kpt-set: ${name} markers with configured values - - image: CHANGE_IMAGE/apply-setters:v0.2.0 + - image: apply-setters:v0.2.0 configMap: app-name: large-app environment: dev @@ -73,30 +73,30 @@ pipeline: memory-limit: 256Mi image-pull-secret: registry-credentials # 4. set-labels — bulk-stamps labels onto every resource - - image: CHANGE_IMAGE/set-labels:v0.2.0 + - image: set-labels:v0.2.0 configMap: app.kubernetes.io/part-of: large-app managed-by: kpt team: platform # 5. set-annotations — bulk-stamps annotations onto every resource - - image: CHANGE_IMAGE/set-annotations:v0.1.4 + - image: set-annotations:v0.1.4 configMap: app.kubernetes.io/managed-by: kpt app.kubernetes.io/part-of: large-app # 6. set-image — updates the nginx container image name and tag - - image: CHANGE_IMAGE/set-image:v0.1.1 + - image: set-image:v0.1.1 configMap: name: nginx newName: docker.io/nginx newTag: latest # 7. search-replace — finds a specific field path and overwrites its value - - image: CHANGE_IMAGE/search-replace:v0.2.0 + - image: search-replace:v0.2.0 configMap: by-path: metadata.annotations.[app.kubernetes.io/managed-by] put-value: kpt # 8. ensure-name-substring — prepends 'large-' to any resource name that # does not already start with that prefix - - image: CHANGE_IMAGE/ensure-name-substring:v0.1.1 + - image: ensure-name-substring:v0.1.1 configMap: prepend: large- selectors: @@ -104,26 +104,26 @@ pipeline: - kind: LimitRange # 9. apply-replacements — propagates field values across resources as defined # in the ApplyReplacements resource inside the package - - image: CHANGE_IMAGE/apply-replacements:v0.1.1 + - image: apply-replacements:v0.1.1 configPath: apply-replacements.yaml # 10. starlark — executes the StarlarkRun script from the package - - image: CHANGE_IMAGE/starlark:v0.4.3 + - image: starlark:v0.4.3 configPath: starlark-run.yaml # 11. upsert-resource — inserts or updates the LimitRange into the package - - image: CHANGE_IMAGE/upsert-resource:v0.2.0 + - image: upsert-resource:v0.2.0 configPath: limitrange.yaml # 12. generate-folders — generates Config Controller Folder resources; no-op # when no HierarchyConfig resources are present in the package - - image: CHANGE_IMAGE/generate-folders:v0.1.1 + - image: generate-folders:v0.1.1 # 13. set-enforcement-action — sets enforcementAction: warn on all Constraint # resources (defined in gatekeeper-constraints.yaml) - - image: CHANGE_IMAGE/set-enforcement-action:v0.1.1 + - image: set-enforcement-action:v0.1.1 configMap: enforcementAction: warn validators: # 14. kubeconform — validates KRM resources against their JSON schemas # (uses baked-in schemas; see https://catalog.kpt.dev/kubeconform/v0.1/) - - image: CHANGE_IMAGE/kubeconform:v0.1.1 + - image: kubeconform:v0.1.1 # 15. gatekeeper — validates resources against ConstraintTemplate/Constraint # policies defined in gatekeeper-constraints.yaml - - image: CHANGE_IMAGE/gatekeeper:v0.2.1 + - image: gatekeeper:v0.2.1 diff --git a/test/performance/packages/small-package/Kptfile b/test/performance/packages/small-package/Kptfile index 88785e51a..d6856e7a6 100644 --- a/test/performance/packages/small-package/Kptfile +++ b/test/performance/packages/small-package/Kptfile @@ -8,9 +8,9 @@ info: description: Generic test package for Porch performance testing pipeline: mutators: - - image: CHANGE_IMAGE/set-namespace:v0.4.1 + - image: set-namespace:v0.4.1 configMap: namespace: CHANGE_NAMESPACE - - image: CHANGE_IMAGE/apply-setters:v0.2.0 + - image: apply-setters:v0.2.0 configMap: image: docker.io/nginx:latest diff --git a/test/performance/performance_suite.go b/test/performance/performance_suite.go index c9aae6d46..2bcb8aa7d 100644 --- a/test/performance/performance_suite.go +++ b/test/performance/performance_suite.go @@ -30,7 +30,6 @@ import ( "syscall" "time" - "github.com/joho/godotenv" porchclient "github.com/kptdev/porch/api/generated/clientset/versioned" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" @@ -77,8 +76,6 @@ var ( giteaUsername = flag.String("gitea-username", "porch", "Gitea username") giteaPassword = flag.String("gitea-password", "secret", "Gitea password") - paddingSize = flag.Int("padding-size", 0, "Size in MB to pad package resources with (0 disables padding)") - retryBackoff = wait.Backoff{ Duration: 50 * time.Millisecond, Steps: 100, @@ -119,11 +116,9 @@ type TestOptions struct { errorRate float64 enableDeletion bool packagePath string - krmFnRegistryURL string giteaURL string giteaUsername string giteaPassword string - paddingSize int } type LogOptions struct { @@ -199,14 +194,6 @@ func (t *PerfTestSuite) initPkgRevMetrics(repoName, pkgName string, revisionNum } } -func getEnvWithDefault(key, defaultValue string) string { - _ = godotenv.Load(filepath.Join("..", "..", ".env")) - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} - func (t *PerfTestSuite) SetupSuite() { if os.Getenv("LOAD_TEST") != "1" && os.Getenv("MAX_PR_TEST=1") != "1" { t.T().Skipf("Skipping performance tests in non-load test environment") @@ -231,11 +218,9 @@ func (t *PerfTestSuite) SetupSuite() { errorRate: *errorRate, enableDeletion: *enableDeletion, packagePath: *packagePath, - krmFnRegistryURL: getEnvWithDefault("PORCH_GHCR_PREFIX_URL", "gcr.io/kptdev/krm-functions-catalog"), giteaURL: *giteaURL, giteaUsername: *giteaUsername, giteaPassword: *giteaPassword, - paddingSize: *paddingSize, } t.logOptions = LogOptions{ @@ -633,7 +618,6 @@ func (t *PerfTestSuite) createPackageResources() map[string]string { resources := t.readPackageResources(t.testOptions.packagePath) if kptfile, ok := resources["Kptfile"]; ok { kptfile = strings.ReplaceAll(kptfile, "CHANGE_NAMESPACE", t.testOptions.namespace) - kptfile = strings.ReplaceAll(kptfile, "CHANGE_IMAGE", t.testOptions.krmFnRegistryURL) resources["Kptfile"] = kptfile } return resources From 3a025ffc92f424c2e4138966c7db2e54d6baee7a Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Wed, 29 Jul 2026 17:04:35 +0200 Subject: [PATCH 09/13] Fix fnrunner and controllers not setting image prefix from .env for local deployment Signed-off-by: Rendre Greyling --- scripts/create-deployment-blueprint.sh | 33 ++++++++++++++++++++++++-- test/performance/README.md | 2 +- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/scripts/create-deployment-blueprint.sh b/scripts/create-deployment-blueprint.sh index 028267b4b..149fc634c 100755 --- a/scripts/create-deployment-blueprint.sh +++ b/scripts/create-deployment-blueprint.sh @@ -187,7 +187,7 @@ for resource in ctx.resource_list["items"]: ' } -function add_image_args_porch_server() { +function add_default_image_prefix() { kpt fn eval ${DESTINATION} \ --image ${STARLARK_IMG} \ --match-kind Deployment \ @@ -199,6 +199,35 @@ for resource in ctx.resource_list['items']: for container in containers: container['args'].append('--default-image-prefix=${GHCR_IMAGE_PREFIX}') " + + kpt fn eval ${DESTINATION} \ + --image ${STARLARK_IMG} \ + --match-kind Deployment \ + --match-name function-runner \ + --match-namespace porch-system \ + -- "source= +for resource in ctx.resource_list['items']: + containers = resource['spec']['template']['spec']['containers'] + for container in containers: + container['command'].append('--default-image-prefix=${GHCR_IMAGE_PREFIX}') +" + + kpt fn eval ${DESTINATION} \ + --image ${STARLARK_IMG} \ + --match-kind Deployment \ + --match-name porch-controllers \ + --match-namespace porch-system \ + -- "source= +for resource in ctx.resource_list['items']: + for container in resource['spec']['template']['spec']['containers']: + if container['name'] == 'porch-controllers': + if container['env'] == None: + container['env'] = [] + container['env'].append({ + 'name': 'DEFAULT_IMAGE_PREFIX', + 'value': '${GHCR_IMAGE_PREFIX}' + }) +" } function disable_fn_runner_warm_up_pod_cache() { @@ -368,7 +397,7 @@ function main() { cp ${PORCH_DIR}/controllers/config/rbac/role.yaml "${DESTINATION}/9-porch-controller-clusterrole.yaml" if [[ -n "${GHCR_IMAGE_PREFIX}" ]]; then - add_image_args_porch_server + add_default_image_prefix fi if [[ "${FN_RUNNER_WARM_UP_POD_CACHE}" == "false" ]]; then diff --git a/test/performance/README.md b/test/performance/README.md index 7cb93ceed..1d8b2b9c5 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -235,7 +235,7 @@ Tests handle `SIGINT`/`SIGTERM` gracefully: in-flight work stops and results col | `-gitea-username` | `porch` | Gitea username | | `-gitea-password` | `secret` | Gitea password | -The KRM function registry URL is configured via `PORCH_GHCR_PREFIX_URL` in the repo root `.env` file. It is applied to porch-server and function-runner at deploy time (`make run-in-kind` reads `.env` automatically). Package `Kptfile` images use short names (for example `set-namespace:v0.4.1`); function-runner resolves them with `--default-image-prefix`. The `CHANGE_NAMESPACE` placeholder in Kptfiles is substituted at test runtime. +The KRM function registry URL is configured via `PORCH_GHCR_PREFIX_URL` in the repo root `.env` file. It is applied at deploy time to porch-server, function-runner, and porch-controllers (`make run-in-kind`, `make run-in-kind-db-cache`, and `make run-in-kind-v1alpha2` all read `.env` automatically via `make deployment-config`). Package `Kptfile` images use short names (for example `set-namespace:v0.4.1`); porch-server and function-runner resolve them with `--default-image-prefix`, and controllers use the `DEFAULT_IMAGE_PREFIX` environment variable. The `CHANGE_NAMESPACE` placeholder in Kptfiles is substituted at test runtime. ## 6. Output Files From e7f3160b9a9525da55f9d005e352531089f42f3f Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Thu, 30 Jul 2026 16:34:01 +0200 Subject: [PATCH 10/13] Address reviews Signed-off-by: Rendre Greyling Co-authored-by: Cursor --- controllers/main.go | 2 + deployments/metrics/pyroscope-deployment.yaml | 2 +- deployments/porch/2-function-runner.yaml | 2 - deployments/porch/3-porch-server.yaml | 4 +- deployments/porch/9-controllers.yaml | 2 - deployments/porch/README.md | 1 + func/server/server.go | 2 + internal/telemetry/metrics.go | 185 ++------------- internal/telemetry/metrics_test.go | 92 +------ internal/telemetry/pprof.go | 2 +- internal/telemetry/pyroscope.go | 103 -------- internal/telemetry/pyroscope_test.go | 129 ---------- pkg/apiserver/config.go | 2 + .../porch/packagerevision_approval.go | 2 +- scripts/deploy-monitoring.sh | 57 ++++- test/performance/csv_generation.go | 2 +- test/performance/driver.go | 68 ++++-- test/performance/driver_v1alpha1.go | 5 +- test/performance/driver_v1alpha2.go | 5 +- test/performance/logger.go | 2 +- test/performance/metrics_utils.go | 224 ++++++++++++++++++ test/performance/performance_suite.go | 22 +- test/performance/porch_metrics_test.go | 4 +- test/performance/types.go | 26 +- 24 files changed, 416 insertions(+), 529 deletions(-) delete mode 100644 internal/telemetry/pyroscope.go delete mode 100644 internal/telemetry/pyroscope_test.go create mode 100644 test/performance/metrics_utils.go diff --git a/controllers/main.go b/controllers/main.go index 2019becec..828f3b5e8 100644 --- a/controllers/main.go +++ b/controllers/main.go @@ -213,6 +213,8 @@ func newManager(scheme *runtime.Scheme) (ctrl.Manager, error) { mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, + // Disable controller-runtime's default :8080 /metrics listener. Port 8080 is reserved for + // optional pprof (PORCH_PPROF_PORT); controller-runtime metrics are exposed on :9464 via OpenTelemetry. Metrics: metricsserver.Options{ BindAddress: "0", }, diff --git a/deployments/metrics/pyroscope-deployment.yaml b/deployments/metrics/pyroscope-deployment.yaml index f1f72ea6f..9ab023772 100644 --- a/deployments/metrics/pyroscope-deployment.yaml +++ b/deployments/metrics/pyroscope-deployment.yaml @@ -1,4 +1,4 @@ -# Copyright 2026 The Nephio Authors +# Copyright 2026 The kpt Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index 2009c9ec7..d5eefcaf6 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -62,8 +62,6 @@ spec: value: none - name: OTEL_SERVICE_NAME value: porch-function-runner - - name: PORCH_PPROF_PORT - value: "8080" ports: - containerPort: 9445 - containerPort: 9464 diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 0a08a3a81..f481601be 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -54,7 +54,7 @@ spec: memory: 256Mi cpu: 250m limits: - memory: 2Gi + memory: 512Mi volumeMounts: - mountPath: /cache name: cache-volume @@ -82,8 +82,6 @@ spec: value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER value: none - - name: PORCH_PPROF_PORT - value: "8080" args: - --function-runner=function-runner:9445 - --cache-directory=/cache diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index 327326af2..2859e4faa 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -80,8 +80,6 @@ spec: value: "9464" # Default value, showing for visibility - name: OTEL_TRACES_EXPORTER value: none - - name: PORCH_PPROF_PORT - value: "8080" envFrom: - configMapRef: name: porch-db-config diff --git a/deployments/porch/README.md b/deployments/porch/README.md index b9c5397b5..64b15bd3a 100644 --- a/deployments/porch/README.md +++ b/deployments/porch/README.md @@ -39,6 +39,7 @@ make deployment-config - `IMAGE_REPO`: Set image repository (default: `ghcr.io/kptdev`) - `ENABLED_RECONCILERS`: Comma-separated list of reconcilers (default: `packagevariants,packagevariantsets,repositories`) - `FN_RUNNER_WARM_UP_POD_CACHE`: Enable/disable pod cache warm-up (default: `true`) +- `PORCH_GHCR_PREFIX_URL`: KRM function catalog registry prefix (from `.env` or environment). Applied to porch-server (`--default-image-prefix`), function-runner (`--default-image-prefix`), and porch-controllers (`DEFAULT_IMAGE_PREFIX`) for all `make run-in-kind*` targets that use `deployment-config`. Examples: diff --git a/func/server/server.go b/func/server/server.go index e9e831e06..5336a556b 100644 --- a/func/server/server.go +++ b/func/server/server.go @@ -245,6 +245,8 @@ func buildFnConfigReconciler(o *options, scheme *runtime.Scheme) (*functionconfi mgr, err := ctrl.NewManager(restCfg, ctrl.Options{ Scheme: scheme, Cache: cacheOpts, + // Disable controller-runtime's default :8080 /metrics listener. Port 8080 is reserved for + // optional pprof (PORCH_PPROF_PORT); controller-runtime metrics are exposed on :9464 via OpenTelemetry. Metrics: metricsserver.Options{ BindAddress: "0", }, diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index d594a3dc4..d30154263 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -16,9 +16,10 @@ package telemetry import ( "context" - "fmt" "time" + "github.com/kptdev/porch/api/porch/v1alpha1" + "github.com/kptdev/porch/api/porch/v1alpha2" "github.com/kptdev/porch/pkg/repository" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -30,9 +31,7 @@ import ( const meterName = "github.com/kptdev/porch" const ( - APIVersionV1Alpha1 = "v1alpha1" - APIVersionV1Alpha2 = "v1alpha2" - ControllerUser = "packagerevision-controller" + ControllerUser = "packagerevision-controller" ResourcePackageRevision = "PackageRevision" ResourcePackageRevisionResources = "PackageRevisionResources" @@ -40,20 +39,13 @@ const ( ) var ( + APIVersionV1Alpha1 = v1alpha1.SchemeGroupVersion.Version + APIVersionV1Alpha2 = v1alpha2.SchemeGroupVersion.Version + apiCallDurationSeconds metric.Float64Histogram - RequestsTotal metric.Float64Counter + requestsTotal metric.Float64Counter prResourceSizeHistogram metric.Int64Histogram prResourceSizeGauge metric.Int64Gauge - - // Performance tests related vars - perfOperationDuration metric.Float64Histogram - perfOperationCounter metric.Float64Counter - perfRepositoryCounter metric.Float64Counter - perfPackageCounter metric.Float64Counter - perfPackageRevisionCounter metric.Float64Counter - perfLifecycleTransitionDuration metric.Float64Histogram - perfTestRunInfoGauge metric.Float64Gauge - perfActiveOperations metric.Float64UpDownCounter ) func InitMetrics() (err error) { @@ -69,15 +61,17 @@ func InitMetrics() (err error) { ), ) if err != nil { - panic(fmt.Sprintf("failed to create porch_api_call_duration_seconds: %v", err)) + klog.Errorf("failed to create porch_api_call_duration_seconds: %v", err) + return } - RequestsTotal, err = m.Float64Counter( + requestsTotal, err = m.Float64Counter( "porch_api_requests_by_user", metric.WithDescription("Total number of requests tracked by BurstCounter, broken down by resource, operation, and user."), ) if err != nil { - panic(fmt.Sprintf("failed to create porch_api_requests_by_user: %v", err)) + klog.Errorf("failed to create porch_api_requests_by_user: %v", err) + return } prResourceSizeHistogram, err = m.Int64Histogram( @@ -101,81 +95,13 @@ func InitMetrics() (err error) { return } - // Performance test related metrics - perfOperationDuration, err = m.Float64Histogram( - "porch_perf_operation_duration_seconds", - metric.WithDescription("Duration of Porch performance test operations in seconds"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30, 60, 120), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_operation_duration_seconds: %v", err)) - } - - perfOperationCounter, err = m.Float64Counter( - "porch_perf_operations_total", - metric.WithDescription("Total number of Porch performance test operations"), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_operations_total: %v", err)) - } - - perfRepositoryCounter, err = m.Float64Counter( - "porch_perf_repositories_created_total", - metric.WithDescription("Total number of repositories created in performance tests"), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_repositories_created_total: %v", err)) - } - - perfPackageCounter, err = m.Float64Counter( - "porch_perf_packages_created_total", - metric.WithDescription("Total number of packages created in performance tests"), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_packages_created_total: %v", err)) - } - - perfPackageRevisionCounter, err = m.Float64Counter( - "porch_perf_package_revisions_total", - metric.WithDescription("Total number of package revisions created in performance tests"), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_package_revisions_total: %v", err)) - } - - perfLifecycleTransitionDuration, err = m.Float64Histogram( - "porch_perf_lifecycle_transition_duration_seconds", - metric.WithDescription("Duration of package lifecycle transitions in seconds"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(0.1, 0.5, 1, 2, 5, 10, 30, 60), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_lifecycle_transition_duration_seconds: %v", err)) - } - - perfTestRunInfoGauge, err = m.Float64Gauge( - "porch_perf_test_run_info", - metric.WithDescription("Information about the current performance test run"), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_test_run_info: %v", err)) - } - - perfActiveOperations, err = m.Float64UpDownCounter( - "porch_perf_active_operations", - metric.WithDescription("Number of currently active operations"), - ) - if err != nil { - panic(fmt.Sprintf("failed to create porch_perf_active_operations: %v", err)) - } - return nil } // Porch server and function runner metric recording functions func RecordAPICallDuration(resource, verb, apiVersion string, durationSeconds float64) { if apiCallDurationSeconds == nil { + klog.Warning("apiCallDurationSeconds is nil - was InitMetrics() called?") return } apiCallDurationSeconds.Record(context.Background(), durationSeconds, @@ -188,14 +114,16 @@ func RecordAPICallDuration(resource, verb, apiVersion string, durationSeconds fl } func RecordRequestCount(ctx context.Context, resource, op, apiVersion string) { - if RequestsTotal == nil { + if requestsTotal == nil { + klog.Warning("requestsTotal is nil - was InitMetrics() called?") return } recordRequestCount(resource, op, apiVersion, getK8sUserName(ctx)) } func RecordControllerRequestCount(resource, op, apiVersion string) { - if RequestsTotal == nil { + if requestsTotal == nil { + klog.Warning("requestsTotal is nil - was InitMetrics() called?") return } recordRequestCount(resource, op, apiVersion, ControllerUser) @@ -208,7 +136,7 @@ func RecordControllerOperation(resource, verb string, start time.Time) { } func recordRequestCount(resource, op, apiVersion, user string) { - RequestsTotal.Add(context.Background(), 1, + requestsTotal.Add(context.Background(), 1, metric.WithAttributes( attribute.String("resource", resource), attribute.String("op", op), @@ -226,6 +154,7 @@ func RecordExternalRepoOperation(ctx context.Context, op string, start time.Time func recordExternalRepoDuration(op string, durationSeconds float64) { if apiCallDurationSeconds == nil { + klog.Warning("apiCallDurationSeconds is nil - was InitMetrics() called?") return } apiCallDurationSeconds.Record(context.Background(), durationSeconds, @@ -237,10 +166,11 @@ func recordExternalRepoDuration(op string, durationSeconds float64) { } func RecordExternalRepoRequestCount(ctx context.Context, op string) { - if RequestsTotal == nil { + if requestsTotal == nil { + klog.Warning("requestsTotal is nil - was InitMetrics() called?") return } - RequestsTotal.Add(context.Background(), 1, + requestsTotal.Add(context.Background(), 1, metric.WithAttributes( attribute.String("resource", ResourceExternalRepo), attribute.String("op", op), @@ -283,77 +213,6 @@ func RecordPackageRevisionResourcesSize(ctx context.Context, prKey repository.Pa prResourceSizeGauge.Record(ctx, resourcesSize, metric.WithAttributeSet(attributes)) } -// Performance test metric recording functions -func PerfTestRecordMetric(operation, apiVersion, repoName, pkgName string, duration time.Duration, err error) { - attrs := metric.WithAttributes( - attribute.String("operation", operation), - attribute.String("api_version", apiVersion), - attribute.String("repository", repoName), - attribute.String("package", pkgName), - attribute.String("status", statusLabel(err)), - ) - ctx := context.Background() - perfOperationDuration.Record(ctx, duration.Seconds(), attrs) - perfOperationCounter.Add(ctx, 1, attrs) -} - -func PerfTestRecordLifecycleTransition(fromState, toState, apiVersion, repoName, pkgName string, duration time.Duration, err error) { - perfLifecycleTransitionDuration.Record(context.Background(), duration.Seconds(), - metric.WithAttributes( - attribute.String("from_state", fromState), - attribute.String("to_state", toState), - attribute.String("api_version", apiVersion), - attribute.String("repository", repoName), - attribute.String("package", pkgName), - attribute.String("status", statusLabel(err)), - ), - ) -} - -func PerfTestRecordPackageRevision(operation string, err error) { - perfPackageRevisionCounter.Add(context.Background(), 1, - metric.WithAttributes( - attribute.String("operation", operation), - attribute.String("status", statusLabel(err)), - ), - ) -} - -func PerfTestSetTestRunInfo(testName, namespace, apiVersion string, startTime time.Time) { - perfTestRunInfoGauge.Record(context.Background(), 1, - metric.WithAttributes( - attribute.String("test_name", testName), - attribute.String("namespace", namespace), - attribute.String("api_version", apiVersion), - attribute.String("start_time", startTime.Format(time.RFC3339)), - ), - ) -} - -func PerfTestRecordActiveOperation(operation, apiVersion string, delta float64) { - perfActiveOperations.Add(context.Background(), delta, - metric.WithAttributes( - attribute.String("operation", operation), - attribute.String("api_version", apiVersion), - ), - ) -} - -func PerfTestIncrementRepositoryCounter() { - perfRepositoryCounter.Add(context.Background(), 1) -} - -func PerfTestIncrementPackageCounter() { - perfPackageCounter.Add(context.Background(), 1) -} - -func statusLabel(err error) string { - if err != nil { - return "error" - } - return "success" -} - func getK8sUserName(ctx context.Context) string { if user, ok := request.UserFrom(ctx); ok { return user.GetName() diff --git a/internal/telemetry/metrics_test.go b/internal/telemetry/metrics_test.go index af6b2c3f9..89427c676 100644 --- a/internal/telemetry/metrics_test.go +++ b/internal/telemetry/metrics_test.go @@ -16,7 +16,6 @@ package telemetry import ( "context" - "errors" "flag" "testing" "time" @@ -207,9 +206,9 @@ func TestRecordRequestCount_UnknownUser(t *testing.T) { func TestRecordRequestCount_NilInstrument(t *testing.T) { setupMetricsTestMeterProvider(t) - before := RequestsTotal - RequestsTotal = nil - t.Cleanup(func() { RequestsTotal = before }) + before := requestsTotal + requestsTotal = nil + t.Cleanup(func() { requestsTotal = before }) assert.NotPanics(t, func() { RecordRequestCount(context.Background(), ResourcePackageRevision, "LIST", APIVersionV1Alpha1) @@ -228,9 +227,9 @@ func TestRecordControllerRequestCount(t *testing.T) { func TestRecordControllerRequestCount_NilInstrument(t *testing.T) { setupMetricsTestMeterProvider(t) - before := RequestsTotal - RequestsTotal = nil - t.Cleanup(func() { RequestsTotal = before }) + before := requestsTotal + requestsTotal = nil + t.Cleanup(func() { requestsTotal = before }) assert.NotPanics(t, func() { RecordControllerRequestCount(ResourcePackageRevision, "UPDATE", APIVersionV1Alpha2) @@ -273,9 +272,9 @@ func TestRecordExternalRepoRequestCount(t *testing.T) { func TestRecordExternalRepoRequestCount_NilInstrument(t *testing.T) { setupMetricsTestMeterProvider(t) - before := RequestsTotal - RequestsTotal = nil - t.Cleanup(func() { RequestsTotal = before }) + before := requestsTotal + requestsTotal = nil + t.Cleanup(func() { requestsTotal = before }) ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "git-user"}) assert.NotPanics(t, func() { @@ -322,79 +321,6 @@ func TestRecordPackageRevisionResourcesSize_VerboseLogging(t *testing.T) { assert.True(t, hasMetric(rm, "porch_package_size_bytes")) } -func TestPerfTestRecordMetric(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestRecordMetric("create", APIVersionV1Alpha2, "repo", "pkg", 100*time.Millisecond, nil) - PerfTestRecordMetric("delete", APIVersionV1Alpha2, "repo", "pkg", 50*time.Millisecond, errors.New("failed")) - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_operation_duration_seconds")) - assert.True(t, hasMetric(rm, "porch_perf_operations_total")) -} - -func TestPerfTestRecordLifecycleTransition(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestRecordLifecycleTransition("Draft", "Published", APIVersionV1Alpha2, "repo", "pkg", 2*time.Second, nil) - PerfTestRecordLifecycleTransition("Published", "Draft", APIVersionV1Alpha2, "repo", "pkg", time.Second, errors.New("failed")) - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_lifecycle_transition_duration_seconds")) -} - -func TestPerfTestRecordPackageRevision(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestRecordPackageRevision("create", nil) - PerfTestRecordPackageRevision("delete", errors.New("failed")) - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_package_revisions_total")) -} - -func TestPerfTestSetTestRunInfo(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestSetTestRunInfo("perf-suite", "default", APIVersionV1Alpha2, time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)) - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_test_run_info")) -} - -func TestPerfTestRecordActiveOperation(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestRecordActiveOperation("create", APIVersionV1Alpha2, 1) - PerfTestRecordActiveOperation("create", APIVersionV1Alpha2, -1) - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_active_operations")) -} - -func TestPerfTestIncrementRepositoryCounter(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestIncrementRepositoryCounter() - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_repositories_created_total")) -} - -func TestPerfTestIncrementPackageCounter(t *testing.T) { - reader := setupMetricsTestMeterProvider(t) - - PerfTestIncrementPackageCounter() - - rm := collectMetricData(t, reader) - assert.True(t, hasMetric(rm, "porch_perf_packages_created_total")) -} - -func TestStatusLabel(t *testing.T) { - assert.Equal(t, "success", statusLabel(nil)) - assert.Equal(t, "error", statusLabel(errors.New("boom"))) -} - func TestGetK8sUserName(t *testing.T) { assert.Equal(t, "", getK8sUserName(context.Background())) diff --git a/internal/telemetry/pprof.go b/internal/telemetry/pprof.go index 854d0eac3..f656f6810 100644 --- a/internal/telemetry/pprof.go +++ b/internal/telemetry/pprof.go @@ -1,4 +1,4 @@ -// Copyright 2025 The Nephio Authors +// Copyright 2025 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/telemetry/pyroscope.go b/internal/telemetry/pyroscope.go deleted file mode 100644 index afb9bebc3..000000000 --- a/internal/telemetry/pyroscope.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2026 The Nephio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package telemetry - -import ( - "os" - "runtime" - - "github.com/grafana/pyroscope-go" - "k8s.io/klog/v2" -) - -const ( - PyroscopeServerEnvVar = "PYROSCOPE_SERVER" - PyroscopeAppNameEnvVar = "PYROSCOPE_APP_NAME" - PyroscopeAuthUserVar = "PYROSCOPE_AUTH_USER" - PyroscopeAuthPassVar = "PYROSCOPE_AUTH_PASSWORD" // #nosec G101 -- env var name, not a credential (nolint:gosec) - PyroscopeLogsEnabledEnvVar = "PYROSCOPE_LOGS_ENABLED" -) - -const defaultPyroscopeAppName = "porch-server" - -type PyroscopeProfiling struct { - stop func() error -} - -func (p *PyroscopeProfiling) Start() { - serverURL := os.Getenv(PyroscopeServerEnvVar) - if serverURL == "" { - return - } - - appName := os.Getenv(PyroscopeAppNameEnvVar) - if appName == "" { - appName = defaultPyroscopeAppName - } - - runtime.SetMutexProfileFraction(1) - runtime.SetBlockProfileRate(1) - - var logger pyroscope.Logger - logsEnabled := os.Getenv(PyroscopeLogsEnabledEnvVar) - if logsEnabled == "true" || logsEnabled == "1" { - logger = pyroscope.StandardLogger - } - - cfg := pyroscope.Config{ - ApplicationName: appName, - ServerAddress: serverURL, - Logger: logger, - Tags: map[string]string{ - "service_name": appName, - }, - ProfileTypes: []pyroscope.ProfileType{ - pyroscope.ProfileCPU, - pyroscope.ProfileAllocObjects, - pyroscope.ProfileAllocSpace, - pyroscope.ProfileInuseObjects, - pyroscope.ProfileInuseSpace, - pyroscope.ProfileGoroutines, - pyroscope.ProfileMutexCount, - pyroscope.ProfileMutexDuration, - pyroscope.ProfileBlockCount, - pyroscope.ProfileBlockDuration, - }, - } - if user := os.Getenv(PyroscopeAuthUserVar); user != "" { - cfg.BasicAuthUser = user - } - if password := os.Getenv(PyroscopeAuthPassVar); password != "" { - cfg.BasicAuthPassword = password - } - - profiler, err := pyroscope.Start(cfg) - if err != nil { - klog.Warningf("Failed to start Pyroscope profiling: %v", err) - return - } - p.stop = profiler.Stop - klog.Infof("Pyroscope continuous profiling started (server=%q, app=%q)", serverURL, appName) -} - -func (p *PyroscopeProfiling) Stop() { - if p.stop != nil { - klog.Infof("Stopping Pyroscope profiler") - if err := p.stop(); err != nil { - klog.Warningf("Pyroscope profiler stop: %v", err) - } - p.stop = nil - } -} diff --git a/internal/telemetry/pyroscope_test.go b/internal/telemetry/pyroscope_test.go deleted file mode 100644 index 0783b0a2c..000000000 --- a/internal/telemetry/pyroscope_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2026 The kpt Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package telemetry - -import ( - "errors" - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPyroscopeProfiling_Start_NoServer(t *testing.T) { - t.Setenv(PyroscopeServerEnvVar, "") - - var p PyroscopeProfiling - p.Start() - - assert.Nil(t, p.stop) -} - -func TestPyroscopeProfiling_Start_InvalidServer(t *testing.T) { - t.Setenv(PyroscopeServerEnvVar, "://invalid-url") - - var p PyroscopeProfiling - p.Start() - - assert.Nil(t, p.stop) -} - -func TestPyroscopeProfiling_Start_Success_DefaultAppName(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ts.Close() - - t.Setenv(PyroscopeServerEnvVar, ts.URL) - t.Setenv(PyroscopeAppNameEnvVar, "") - t.Setenv(PyroscopeAuthUserVar, "") - t.Setenv(PyroscopeAuthPassVar, "") - t.Setenv(PyroscopeLogsEnabledEnvVar, "") - - var p PyroscopeProfiling - p.Start() - require.NotNil(t, p.stop) - t.Cleanup(func() { p.Stop() }) -} - -func TestPyroscopeProfiling_Start_Success_CustomConfig(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ts.Close() - - t.Setenv(PyroscopeServerEnvVar, ts.URL) - t.Setenv(PyroscopeAppNameEnvVar, "custom-app") - t.Setenv(PyroscopeAuthUserVar, "user") - t.Setenv(PyroscopeAuthPassVar, "pass") - t.Setenv(PyroscopeLogsEnabledEnvVar, "1") - - var p PyroscopeProfiling - p.Start() - require.NotNil(t, p.stop) - p.Stop() - assert.Nil(t, p.stop) -} - -func TestPyroscopeProfiling_Start_LogsEnabledTrue(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ts.Close() - - t.Setenv(PyroscopeServerEnvVar, ts.URL) - t.Setenv(PyroscopeAppNameEnvVar, "logs-app") - t.Setenv(PyroscopeLogsEnabledEnvVar, "true") - - var p PyroscopeProfiling - p.Start() - require.NotNil(t, p.stop) - p.Stop() -} - -func TestPyroscopeProfiling_Stop_NoProfiler(t *testing.T) { - var p PyroscopeProfiling - assert.NotPanics(t, func() { p.Stop() }) -} - -func TestPyroscopeProfiling_Stop_WithError(t *testing.T) { - var p PyroscopeProfiling - p.stop = func() error { - return errors.New("stop failed") - } - - assert.NotPanics(t, func() { p.Stop() }) - assert.Nil(t, p.stop) -} - -func TestPyroscopeProfiling_Stop_Success(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ts.Close() - - t.Setenv(PyroscopeServerEnvVar, ts.URL) - t.Setenv(PyroscopeAppNameEnvVar, fmt.Sprintf("stop-test-%d", time.Now().UnixNano())) - - var p PyroscopeProfiling - p.Start() - require.NotNil(t, p.stop) - assert.NotPanics(t, func() { p.Stop() }) - assert.Nil(t, p.stop) -} diff --git a/pkg/apiserver/config.go b/pkg/apiserver/config.go index 97c83b855..6eb921f85 100644 --- a/pkg/apiserver/config.go +++ b/pkg/apiserver/config.go @@ -277,6 +277,8 @@ func (c *completedConfig) buildManager(restConfig *rest.Config, scheme *runtime. }, }, HealthProbeBindAddress: probePort, + // Disable controller-runtime's default :8080 /metrics listener. Port 8080 is reserved for + // optional pprof (PORCH_PPROF_PORT); controller-runtime metrics are exposed on :9464 via OpenTelemetry. Metrics: metricsserver.Options{ BindAddress: "0", }, diff --git a/pkg/registry/porch/packagerevision_approval.go b/pkg/registry/porch/packagerevision_approval.go index d0d9616a9..2f9f722a1 100644 --- a/pkg/registry/porch/packagerevision_approval.go +++ b/pkg/registry/porch/packagerevision_approval.go @@ -1,4 +1,4 @@ -// Copyright 2022 The kpt Authors +// Copyright 2022, 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/scripts/deploy-monitoring.sh b/scripts/deploy-monitoring.sh index f0ec18a7a..2c57b38a3 100755 --- a/scripts/deploy-monitoring.sh +++ b/scripts/deploy-monitoring.sh @@ -36,6 +36,7 @@ PORCH_PPROF_DEPLOYMENTS=( function-runner:porch-function-runner porch-controllers:porch-controllers ) +PORCH_PPROF_PORT="${PORCH_PPROF_PORT:-8080}" PROMETHEUS_LOCAL_PORT="${PROMETHEUS_LOCAL_PORT:-9092}" PROMETHEUS_CONTAINER_PORT="${PROMETHEUS_CONTAINER_PORT:-9090}" PROMETHEUS_NODEPORT="${PROMETHEUS_NODEPORT:-30091}" @@ -50,7 +51,6 @@ JAEGER_UI_CONTAINER_PORT="${JAEGER_UI_CONTAINER_PORT:-16686}" GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-porch}" GRAFANA_ADMIN_PW="${GRAFANA_ADMIN_PW:-}" -[[ -z $GRAFANA_ADMIN_PW ]] && GRAFANA_ADMIN_PW="$(date +%s | shasum -a 256 | base64 | head -c 15)" DOCKERHUB_MIRROR="${DOCKERHUB_MIRROR:-docker.io}" KRM_FN_REGISTRY_URL="${KRM_FN_REGISTRY_URL:-ghcr.io/kptdev/krm-functions-catalog}" @@ -130,6 +130,30 @@ is_pyroscope_deployed() { kubectl get deployment pyroscope -n "$NAMESPACE" &> /dev/null } +# When GRAFANA_ADMIN_PW is not set in the environment, reuse the cluster secret so +# re-applying manifests does not rotate the password while Grafana keeps the old one. +resolve_grafana_admin_creds() { + if [[ -n $GRAFANA_ADMIN_PW ]]; then + return 0 + fi + + if kubectl get secret grafana-admin-creds -n "$NAMESPACE" &> /dev/null; then + GRAFANA_ADMIN_USER="$( + kubectl get secret grafana-admin-creds -n "$NAMESPACE" \ + -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d + )" + GRAFANA_ADMIN_PW="$( + kubectl get secret grafana-admin-creds -n "$NAMESPACE" \ + -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d + )" + log_info "Using existing Grafana admin credentials from secret grafana-admin-creds" + return 0 + fi + + GRAFANA_ADMIN_PW="$(date +%s | shasum -a 256 | base64 | head -c 15)" + log_info "Generated new Grafana admin password (will be stored in grafana-admin-creds)" +} + prepare_manifests() { local temp_dir temp_dir=$(mktemp -d) @@ -241,6 +265,7 @@ deploy_base_stack() { log_info "Deploying base monitoring stack (Prometheus, Grafana, Postgres Exporter)..." log_info "Rendering manifests with kpt..." + resolve_grafana_admin_creds apply_base_configmaps local manifests_dir @@ -309,6 +334,7 @@ deploy_pyroscope() { wait_for_deployment pyroscope wait_for_deployment alloy + enable_porch_pprof_port enable_porch_pprof_annotations log_info "Pyroscope and Alloy deployed successfully" @@ -355,6 +381,30 @@ disable_porch_trace_export() { done } +enable_porch_pprof_port() { + local entry deployment + for entry in "${PORCH_PPROF_DEPLOYMENTS[@]}"; do + deployment="${entry%%:*}" + if kubectl get deployment "$deployment" -n "$PORCH_NAMESPACE" &> /dev/null; then + log_info "Enabling pprof on ${PORCH_NAMESPACE}/${deployment} (PORCH_PPROF_PORT=${PORCH_PPROF_PORT})..." + kubectl set env deployment/"$deployment" -n "$PORCH_NAMESPACE" "PORCH_PPROF_PORT=${PORCH_PPROF_PORT}" + else + log_info "Skipping pprof port for ${PORCH_NAMESPACE}/${deployment} (not deployed)" + fi + done +} + +disable_porch_pprof_port() { + local entry deployment + for entry in "${PORCH_PPROF_DEPLOYMENTS[@]}"; do + deployment="${entry%%:*}" + if kubectl get deployment "$deployment" -n "$PORCH_NAMESPACE" &> /dev/null; then + log_info "Disabling pprof on ${PORCH_NAMESPACE}/${deployment}..." + kubectl set env deployment/"$deployment" -n "$PORCH_NAMESPACE" "PORCH_PPROF_PORT-" + fi + done +} + enable_porch_pprof_annotations() { local entry deployment service_name for entry in "${PORCH_PPROF_DEPLOYMENTS[@]}"; do @@ -433,6 +483,10 @@ stop_port_forwards() { get_service_urls() { log_info "Getting service URLs..." + if is_base_deployed; then + resolve_grafana_admin_creds + fi + log_info "Setting up port forwarding..." stop_port_forwards sleep 2 @@ -504,6 +558,7 @@ cleanup() { log_info "Stopping port forwarding..." stop_port_forwards disable_porch_trace_export + disable_porch_pprof_port disable_porch_pprof_annotations if kubectl get namespace "$NAMESPACE" &> /dev/null; then diff --git a/test/performance/csv_generation.go b/test/performance/csv_generation.go index 2d8815586..a12145e37 100644 --- a/test/performance/csv_generation.go +++ b/test/performance/csv_generation.go @@ -1,4 +1,4 @@ -// Copyright 2026 The Kpt Authors +// Copyright 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/performance/driver.go b/test/performance/driver.go index df658d340..d1d763ec6 100644 --- a/test/performance/driver.go +++ b/test/performance/driver.go @@ -20,7 +20,6 @@ import ( "time" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" - "github.com/kptdev/porch/internal/telemetry" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,11 +54,7 @@ type deletionCandidateExtractor func(client.Object) (DeletionCandidate, bool) func (b *baseDriver) createPackageRevision(obj client.Object, repoName, pkgName string, revisionNum int) error { t := b.suite start := time.Now() - if t.enablePrometheus { - apiVersion := string(t.testOptions.apiVersion) - telemetry.PerfTestRecordActiveOperation(pkgRevCreate, apiVersion, 1) - defer telemetry.PerfTestRecordActiveOperation(pkgRevCreate, apiVersion, -1) - } + defer t.trackPerfActiveOperation(pkgRevCreate)() err := retry.RetryOnConflict(retryBackoff, func() error { return t.client.Create(t.ctx, obj) @@ -73,9 +68,7 @@ func (b *baseDriver) createPackageRevision(obj client.Object, repoName, pkgName Timestamp: start, }) t.recordPerfMetric(pkgRevCreate, repoName, pkgName, duration, err) - if t.enablePrometheus { - telemetry.PerfTestRecordPackageRevision(pkgRevCreate, err) - } + t.recordPerfPackageRevision(pkgRevCreate, err) return err } @@ -170,8 +163,8 @@ func (b *baseDriver) deletePackageRevision( Timestamp: start, }) t.recordPerfMetric(pkgRevProposeDeletion, repoName, pkgName, duration, err) - if t.enablePrometheus && shouldPropose { - telemetry.PerfTestRecordLifecycleTransition(initialLifecycle, behavior.deletionProposed, string(t.testOptions.apiVersion), repoName, pkgName, duration, err) + if shouldPropose { + t.recordPerfLifecycleTransition(initialLifecycle, behavior.deletionProposed, repoName, pkgName, duration, err) } if err != nil { return err @@ -203,8 +196,8 @@ func (b *baseDriver) deletePackageRevision( Timestamp: start, }) t.recordPerfMetric(pkgRevDelete, repoName, pkgName, duration, err) - if t.enablePrometheus && shouldPropose { - telemetry.PerfTestRecordLifecycleTransition(behavior.deletionProposed, "deleted", string(t.testOptions.apiVersion), repoName, pkgName, duration, err) + if shouldPropose { + t.recordPerfLifecycleTransition(behavior.deletionProposed, "deleted", repoName, pkgName, duration, err) } return err @@ -285,9 +278,7 @@ func (b *baseDriver) proposePackage( return setProposed(obj) }) b.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevPropose, start, err) - if t.enablePrometheus { - telemetry.PerfTestRecordLifecycleTransition(initialLifecycle, proposedLifecycle, string(t.testOptions.apiVersion), repoName, pkgName, time.Since(start), err) - } + t.recordPerfLifecycleTransition(initialLifecycle, proposedLifecycle, repoName, pkgName, time.Since(start), err) return err } @@ -312,18 +303,55 @@ func (b *baseDriver) approvePackage( return publish(obj) }) b.recordPkgRevOperation(repoName, pkgName, revisionNum, pkgRevPublished, start, err) - if t.enablePrometheus { - telemetry.PerfTestRecordLifecycleTransition(proposedLifecycle, publishedLifecycle, string(t.testOptions.apiVersion), repoName, pkgName, time.Since(start), err) - } + t.recordPerfLifecycleTransition(proposedLifecycle, publishedLifecycle, repoName, pkgName, time.Since(start), err) return err } +func (t *PerfTestSuite) trackPerfActiveOperation(operation string) func() { + if !t.enablePrometheus { + return func() {} + } + apiVersion := string(t.testOptions.apiVersion) + RecordPerfActiveOperation(operation, apiVersion, 1) + return func() { + RecordPerfActiveOperation(operation, apiVersion, -1) + } +} + +func (t *PerfTestSuite) recordPerfPackageRevision(operation string, err error) { + if !t.enablePrometheus { + return + } + RecordPerfPackageRevision(operation, err) +} + +func (t *PerfTestSuite) recordPerfLifecycleTransition(fromState, toState, repoName, pkgName string, duration time.Duration, err error) { + if !t.enablePrometheus { + return + } + RecordPerfLifecycleTransition(fromState, toState, string(t.testOptions.apiVersion), repoName, pkgName, duration, err) +} + +func (t *PerfTestSuite) incrementPerfPackageCounter() { + if !t.enablePrometheus { + return + } + IncrementPerfPackageCounter() +} + func (t *PerfTestSuite) recordPerfMetric(operation, repoName, pkgName string, duration time.Duration, err error) { if !t.enablePrometheus { return } - telemetry.PerfTestRecordMetric(operation, string(t.testOptions.apiVersion), repoName, pkgName, duration, err) + RecordPerfMetric(operation, string(t.testOptions.apiVersion), repoName, pkgName, duration, err) +} + +func (t *PerfTestSuite) incrementPerfRepositoryCounter() { + if !t.enablePrometheus { + return + } + IncrementPerfRepositoryCounter() } func packageRevisionCRDName(repo, pkg, workspace string) string { diff --git a/test/performance/driver_v1alpha1.go b/test/performance/driver_v1alpha1.go index 66e283565..3f951e373 100644 --- a/test/performance/driver_v1alpha1.go +++ b/test/performance/driver_v1alpha1.go @@ -19,7 +19,6 @@ import ( "time" porchv1alpha1 "github.com/kptdev/porch/api/porch/v1alpha1" - "github.com/kptdev/porch/internal/telemetry" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -76,9 +75,7 @@ func (d *v1alpha1Driver) DoLifecycle(repoName, pkgName string, revisionNum int) }, }, } - if t.enablePrometheus { - telemetry.PerfTestIncrementPackageCounter() - } + t.incrementPerfPackageCounter() } else if latestPR != nil { taskList = []porchv1alpha1.Task{ { diff --git a/test/performance/driver_v1alpha2.go b/test/performance/driver_v1alpha2.go index 99174319a..8cf461ce4 100644 --- a/test/performance/driver_v1alpha2.go +++ b/test/performance/driver_v1alpha2.go @@ -19,7 +19,6 @@ import ( "time" porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2" - "github.com/kptdev/porch/internal/telemetry" pkgerrors "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -92,9 +91,7 @@ func (d *v1alpha2Driver) DoLifecycle(repoName, pkgName string, revisionNum int) Site: "https://kpt.dev/", }, } - if t.enablePrometheus { - telemetry.PerfTestIncrementPackageCounter() - } + t.incrementPerfPackageCounter() } else if latestPR != nil { pkgRev.Spec.Source = &porchv1alpha2.PackageSource{ CopyFrom: &porchv1alpha2.PackageRevisionRef{ diff --git a/test/performance/logger.go b/test/performance/logger.go index e10a67053..eb252d76a 100644 --- a/test/performance/logger.go +++ b/test/performance/logger.go @@ -1,4 +1,4 @@ -// Copyright 2026 The kpt Authors +// Copyright 2025-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/test/performance/metrics_utils.go b/test/performance/metrics_utils.go new file mode 100644 index 000000000..722509a17 --- /dev/null +++ b/test/performance/metrics_utils.go @@ -0,0 +1,224 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "k8s.io/klog/v2" +) + +const perfMeterName = "github.com/kptdev/porch" + +var ( + perfOperationDuration metric.Float64Histogram + perfOperationCounter metric.Float64Counter + perfRepositoryCounter metric.Float64Counter + perfPackageCounter metric.Float64Counter + perfPackageRevisionCounter metric.Float64Counter + perfLifecycleTransitionDuration metric.Float64Histogram + perfTestRunInfoGauge metric.Float64Gauge + perfActiveOperations metric.Float64UpDownCounter +) + +// InitPerfMetrics registers OpenTelemetry instruments used by performance tests. +// Call after telemetry.SetupOpenTelemetry so the meter provider is configured. +func InitPerfMetrics() (err error) { + m := otel.Meter(perfMeterName) + + perfOperationDuration, err = m.Float64Histogram( + "porch_perf_operation_duration_seconds", + metric.WithDescription("Duration of Porch performance test operations in seconds"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30, 60, 120), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_operation_duration_seconds: %v", err) + return + } + + perfOperationCounter, err = m.Float64Counter( + "porch_perf_operations_total", + metric.WithDescription("Total number of Porch performance test operations"), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_operations_total: %v", err) + return + } + + perfRepositoryCounter, err = m.Float64Counter( + "porch_perf_repositories_created_total", + metric.WithDescription("Total number of repositories created in performance tests"), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_repositories_created_total: %v", err) + return + } + + perfPackageCounter, err = m.Float64Counter( + "porch_perf_packages_created_total", + metric.WithDescription("Total number of packages created in performance tests"), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_packages_created_total: %v", err) + return + } + + perfPackageRevisionCounter, err = m.Float64Counter( + "porch_perf_package_revisions_total", + metric.WithDescription("Total number of package revisions created in performance tests"), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_package_revisions_total: %v", err) + return + } + + perfLifecycleTransitionDuration, err = m.Float64Histogram( + "porch_perf_lifecycle_transition_duration_seconds", + metric.WithDescription("Duration of package lifecycle transitions in seconds"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.1, 0.5, 1, 2, 5, 10, 30, 60), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_lifecycle_transition_duration_seconds: %v", err) + return + } + + perfTestRunInfoGauge, err = m.Float64Gauge( + "porch_perf_test_run_info", + metric.WithDescription("Information about the current performance test run"), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_test_run_info: %v", err) + return + } + + perfActiveOperations, err = m.Float64UpDownCounter( + "porch_perf_active_operations", + metric.WithDescription("Number of currently active operations"), + ) + if err != nil { + klog.Errorf("failed to create porch_perf_active_operations: %v", err) + return + } + + return nil +} + +func RecordPerfMetric(operation, apiVersion, repoName, pkgName string, duration time.Duration, err error) { + if perfOperationDuration == nil { + klog.Warning("perfOperationDuration is nil - was InitPerfMetrics() called?") + return + } + if perfOperationCounter == nil { + klog.Warning("perfOperationCounter is nil - was InitPerfMetrics() called?") + return + } + attrs := metric.WithAttributes( + attribute.String("operation", operation), + attribute.String("api_version", apiVersion), + attribute.String("repository", repoName), + attribute.String("package", pkgName), + attribute.String("status", perfStatusLabel(err)), + ) + ctx := context.Background() + perfOperationDuration.Record(ctx, duration.Seconds(), attrs) + perfOperationCounter.Add(ctx, 1, attrs) +} + +func RecordPerfLifecycleTransition(fromState, toState, apiVersion, repoName, pkgName string, duration time.Duration, err error) { + if perfLifecycleTransitionDuration == nil { + klog.Warning("perfLifecycleTransitionDuration is nil - was InitPerfMetrics() called?") + return + } + perfLifecycleTransitionDuration.Record(context.Background(), duration.Seconds(), + metric.WithAttributes( + attribute.String("from_state", fromState), + attribute.String("to_state", toState), + attribute.String("api_version", apiVersion), + attribute.String("repository", repoName), + attribute.String("package", pkgName), + attribute.String("status", perfStatusLabel(err)), + ), + ) +} + +func RecordPerfPackageRevision(operation string, err error) { + if perfPackageRevisionCounter == nil { + klog.Warning("perfPackageRevisionCounter is nil - was InitPerfMetrics() called?") + return + } + perfPackageRevisionCounter.Add(context.Background(), 1, + metric.WithAttributes( + attribute.String("operation", operation), + attribute.String("status", perfStatusLabel(err)), + ), + ) +} + +func SetPerfTestRunInfo(testName, namespace, apiVersion string, startTime time.Time) { + if perfTestRunInfoGauge == nil { + klog.Warning("perfTestRunInfoGauge is nil - was InitPerfMetrics() called?") + return + } + perfTestRunInfoGauge.Record(context.Background(), 1, + metric.WithAttributes( + attribute.String("test_name", testName), + attribute.String("namespace", namespace), + attribute.String("api_version", apiVersion), + attribute.String("start_time", startTime.Format(time.RFC3339)), + ), + ) +} + +func RecordPerfActiveOperation(operation, apiVersion string, delta float64) { + if perfActiveOperations == nil { + klog.Warning("perfActiveOperations is nil - was InitPerfMetrics() called?") + return + } + perfActiveOperations.Add(context.Background(), delta, + metric.WithAttributes( + attribute.String("operation", operation), + attribute.String("api_version", apiVersion), + ), + ) +} + +func IncrementPerfRepositoryCounter() { + if perfRepositoryCounter == nil { + klog.Warning("perfRepositoryCounter is nil - was InitPerfMetrics() called?") + return + } + perfRepositoryCounter.Add(context.Background(), 1) +} + +func IncrementPerfPackageCounter() { + if perfPackageCounter == nil { + klog.Warning("perfPackageCounter is nil - was InitPerfMetrics() called?") + return + } + perfPackageCounter.Add(context.Background(), 1) +} + +func perfStatusLabel(err error) string { + if err != nil { + return "error" + } + return "success" +} diff --git a/test/performance/performance_suite.go b/test/performance/performance_suite.go index 2bcb8aa7d..e73a88927 100644 --- a/test/performance/performance_suite.go +++ b/test/performance/performance_suite.go @@ -1,4 +1,4 @@ -// Copyright 2026 The kpt Authors +// Copyright 2025-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -62,7 +62,7 @@ var ( packageParallelism = flag.Int("package-parallelism", 1, "Number of packages to create in parallel per repository") errorRate = flag.Float64("error-rate", 0.1, "Maximum percentage of package revisions allowed to fail lifecycle transition") enableDeletion = flag.Bool("enable-deletion", false, "Enable deletion of package revisions at the end of the test") - enablePrometheus = flag.Bool("enable-prometheus", false, "Enable Prometheus metrics server on port 9091") + enablePrometheus = flag.Bool("enable-prometheus", false, "Enable Prometheus metrics server on port 9095") metricsLogFile = flag.String("metrics-log-prefix", "porch-metrics", "Prefix for the timestamped metrics log file") resultsFile = flag.String("results-file", "load_test_results.txt", "File name for test results") @@ -96,6 +96,7 @@ type PerfTestSuite struct { otelResources *telemetry.OTelResources enablePrometheus bool lifecycleDriver LifecycleDriver + testMode TestMode metrics map[string]TestMetrics metricsMutex sync.RWMutex @@ -105,6 +106,11 @@ type PerfTestSuite struct { csvOptions CSVOptions } +// IsTestModeEnabled returns true if the suite is running under the given TestMode. +func (t *PerfTestSuite) IsTestModeEnabled(mode TestMode) bool { + return t.testMode == mode +} + type TestOptions struct { apiVersion PorchAPIVersion namespace string @@ -195,7 +201,8 @@ func (t *PerfTestSuite) initPkgRevMetrics(repoName, pkgName string, revisionNum } func (t *PerfTestSuite) SetupSuite() { - if os.Getenv("LOAD_TEST") != "1" && os.Getenv("MAX_PR_TEST=1") != "1" { + t.testMode = ActiveTestMode() + if t.testMode == TestModeNone { t.T().Skipf("Skipping performance tests in non-load test environment") } @@ -286,9 +293,12 @@ func (t *PerfTestSuite) SetupSuite() { if err != nil { t.T().Fatalf("Failed to set up OpenTelemetry: %v", err) } + if err := InitPerfMetrics(); err != nil { + t.T().Fatalf("Failed to initialize performance metrics: %v", err) + } t.otelResources = otelRes t.T().Logf("OTel metrics server started on port %v", prometheusPort) - telemetry.PerfTestSetTestRunInfo("porch-performance-test", t.testOptions.namespace, string(apiVersion), time.Now()) + SetPerfTestRunInfo("porch-performance-test", t.testOptions.namespace, string(apiVersion), time.Now()) } t.T().Logf(" Running load test with:") @@ -496,9 +506,7 @@ func (t *PerfTestSuite) createAndSetupRepo(repoName string) { return } - if t.enablePrometheus { - telemetry.PerfTestIncrementRepositoryCounter() - } + t.incrementPerfRepositoryCounter() startWait := time.Now() err = t.waitForRepository(repoName, 60*time.Second) duration = time.Since(startWait) diff --git a/test/performance/porch_metrics_test.go b/test/performance/porch_metrics_test.go index ac8b03d81..94808eb2e 100644 --- a/test/performance/porch_metrics_test.go +++ b/test/performance/porch_metrics_test.go @@ -47,7 +47,7 @@ func TestPerf(t *testing.T) { } func (t *PerformanceTests) TestPorchScalePerformance() { - if os.Getenv("LOAD_TEST") != "1" { + if !t.IsTestModeEnabled(TestModeLoad) { t.T().Skipf("LOAD_TEST != 1: Skipping performance tests in non-load test environment") } @@ -98,7 +98,7 @@ func (t *PerformanceTests) TestPorchScalePerformance() { func (t *PerformanceTests) TestIncreasePRsPerformance() { maxPkgRevNum := math.MaxInt - if os.Getenv("MAX_PR_TEST") != "1" { + if !t.IsTestModeEnabled(TestModeMaxPR) { t.T().Skipf("MAX_PR_TEST != 1: Skipping performance tests in non-load test environment") } diff --git a/test/performance/types.go b/test/performance/types.go index 0c9969168..3e222156e 100644 --- a/test/performance/types.go +++ b/test/performance/types.go @@ -1,4 +1,4 @@ -// Copyright 2026 The kpt Authors +// Copyright 2025-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,9 +16,33 @@ package metrics import ( "fmt" + "os" "time" ) +// TestMode identifies which load-test mode the performance suite is running in. +type TestMode string + +const ( + // TestModeNone means no load-test environment variable is set; tests are skipped. + TestModeNone TestMode = "" + // TestModeLoad is enabled by setting LOAD_TEST=1. + TestModeLoad TestMode = "load" + // TestModeMaxPR is enabled by setting MAX_PR_TEST=1. + TestModeMaxPR TestMode = "max_pr" +) + +// ActiveTestMode reads the environment and returns the active TestMode. +func ActiveTestMode() TestMode { + if os.Getenv("LOAD_TEST") == "1" { + return TestModeLoad + } + if os.Getenv("MAX_PR_TEST") == "1" { + return TestModeMaxPR + } + return TestModeNone +} + // PorchAPIVersion identifies which Porch PackageRevision API the performance test uses. type PorchAPIVersion string From f64f47e0110b04b3e380c0f0846287b2c984bd97 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Sun, 2 Aug 2026 15:58:34 +0200 Subject: [PATCH 11/13] Address further review comments Signed-off-by: Rendre Greyling --- go.mod | 6 --- go.sum | 4 -- internal/telemetry/metrics.go | 31 ++++++++++++--- internal/telemetry/otel.go | 5 +-- test/performance/performance_suite.go | 54 +++++++-------------------- 5 files changed, 42 insertions(+), 58 deletions(-) diff --git a/go.mod b/go.mod index 0828f111f..8af4247ef 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/go-containerregistry v0.21.6 github.com/google/uuid v1.6.0 - github.com/grafana/pyroscope-go v1.4.1 github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 github.com/kptdev/kpt v1.0.0-beta.67 @@ -73,11 +72,6 @@ require ( require sigs.k8s.io/cli-utils v0.37.2 // indirect -require ( - github.com/grafana/pyroscope-go/godeltaprof v0.1.11 // indirect - sigs.k8s.io/cli-utils v0.37.2 // indirect -) - require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go/auth v0.20.0 // indirect diff --git a/go.sum b/go.sum index 2d81cd6b3..d924893e3 100644 --- a/go.sum +++ b/go.sum @@ -229,10 +229,6 @@ github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/grafana/pyroscope-go v1.4.1 h1:SKvuZz1qTFpNXjHY3XGlm92mS9wJeCI/TZR42XcDs9w= -github.com/grafana/pyroscope-go v1.4.1/go.mod h1:WQY21vHNiD2o/icxqVPM0AmofabycbhnPPHoV1/4X6s= -github.com/grafana/pyroscope-go/godeltaprof v0.1.11 h1:el5LYpXissAiCKZ5/6yjlr6mhYVV6Cp5lahTocxraXM= -github.com/grafana/pyroscope-go/godeltaprof v0.1.11/go.mod h1:jl1V8M4cWsXciROCPIDDG7CtjSjT/ECbp6eLVuMxYRI= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index d30154263..67c274dc4 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -36,6 +36,12 @@ const ( ResourcePackageRevision = "PackageRevision" ResourcePackageRevisionResources = "PackageRevisionResources" ResourceExternalRepo = "ExternalRepo" + + apiCallDurationStartingBucket = 0.001 + apiCallDurationBucketCount = 16 + + packageSizeStartingBucket = 1024 + packageSizeBucketCount = 21 // doubling boundaries after the initial zero bucket ) var ( @@ -55,10 +61,7 @@ func InitMetrics() (err error) { "porch_api_call_duration_seconds", metric.WithDescription("Duration of porch API calls in seconds."), metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries( - 0.001, 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, - 0.256, 0.512, 1.024, 2.048, 4.096, 8.192, 16.384, 32.768, - ), + metric.WithExplicitBucketBoundaries(doublingBucketBoundaries(apiCallDurationStartingBucket, apiCallDurationBucketCount)...), ) if err != nil { klog.Errorf("failed to create porch_api_call_duration_seconds: %v", err) @@ -78,7 +81,7 @@ func InitMetrics() (err error) { "porch_package_size_bytes", metric.WithUnit("By"), metric.WithDescription("Distribution of package revision resources' file size, in bytes"), - metric.WithExplicitBucketBoundaries(0, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824), + metric.WithExplicitBucketBoundaries(packageSizeBucketBoundaries()...), ) if err != nil { klog.Errorf("failed to create porch_package_size_bytes histogram: %v", err) @@ -213,6 +216,24 @@ func RecordPackageRevisionResourcesSize(ctx context.Context, prKey repository.Pa prResourceSizeGauge.Record(ctx, resourcesSize, metric.WithAttributeSet(attributes)) } +func doublingBucketBoundaries(start float64, count int) []float64 { + buckets := make([]float64, count) + v := start + for i := range buckets { + buckets[i] = v + v *= 2 + } + return buckets +} + +func packageSizeBucketBoundaries() []float64 { + doubled := doublingBucketBoundaries(float64(packageSizeStartingBucket), packageSizeBucketCount) + buckets := make([]float64, 1+len(doubled)) + buckets[0] = 0 + copy(buckets[1:], doubled) + return buckets +} + func getK8sUserName(ctx context.Context) string { if user, ok := request.UserFrom(ctx); ok { return user.GetName() diff --git a/internal/telemetry/otel.go b/internal/telemetry/otel.go index 8d5203ea0..5090bb887 100644 --- a/internal/telemetry/otel.go +++ b/internal/telemetry/otel.go @@ -120,9 +120,8 @@ func SetupOpenTelemetry(ctx context.Context) (*OTelResources, error) { return nil, err } - prof := &Profiling{} - prof.Start() - res.profiling = prof + res.profiling = &Profiling{} + res.profiling.Start() http.DefaultTransport = otelhttp.NewTransport(http.DefaultTransport) http.DefaultClient.Transport = http.DefaultTransport diff --git a/test/performance/performance_suite.go b/test/performance/performance_suite.go index e73a88927..05d8929fc 100644 --- a/test/performance/performance_suite.go +++ b/test/performance/performance_suite.go @@ -209,9 +209,7 @@ func (t *PerfTestSuite) SetupSuite() { flag.Parse() apiVersion, err := ParsePorchAPIVersion(*porchAPIVersion) - if err != nil { - t.T().Fatalf("Invalid api-version flag: %v", err) - } + t.Require().NoError(err, "invalid api-version flag") t.metrics = make(map[string]TestMetrics) t.testOptions = TestOptions{ @@ -243,30 +241,20 @@ func (t *PerfTestSuite) SetupSuite() { } logger, err := t.NewTestLogger(t.logOptions.metricsLogFile, apiVersion) - if err != nil { - t.T().Fatalf("Failed to create logger: %v", err) - } + t.Require().NoError(err, "failed to create logger") resultsLogger, err := t.NewResultsLogger(t.logOptions.resultsFile, t.logOptions.fullLogFile) - if err != nil { - t.T().Fatalf("Failed to create results logger: %v", err) - } + t.Require().NoError(err, "failed to create results logger") resultsLogger.LogTestConfig(apiVersion, t.testOptions, t.enablePrometheus) cfg, err := config.GetConfig() - if err != nil { - t.T().Fatalf("Failed to get config: %v", err) - } + t.Require().NoError(err, "failed to get config") c, err := client.New(cfg, client.Options{Scheme: scheme}) - if err != nil { - t.T().Fatalf("Failed to create client: %v", err) - } + t.Require().NoError(err, "failed to create client") clientSet, err := porchclient.NewForConfig(cfg) - if err != nil { - t.T().Fatalf("Failed to create Porch clientset: %v", err) - } + t.Require().NoError(err, "failed to create Porch clientset") t.ctx, t.cancel = context.WithCancel(context.Background()) t.setupSignalHandler() @@ -277,25 +265,15 @@ func (t *PerfTestSuite) SetupSuite() { t.enablePrometheus = *enablePrometheus lifecycleDriver, err := NewLifecycleDriver(apiVersion, t) - if err != nil { - t.T().Fatalf("Failed to create lifecycle driver: %v", err) - } + t.Require().NoError(err, "failed to create lifecycle driver") t.lifecycleDriver = lifecycleDriver if t.enablePrometheus { - if err := os.Setenv("OTEL_METRICS_EXPORTER", "prometheus"); err != nil { - t.T().Fatalf("Failed to set OTEL_METRICS_EXPORTER: %v", err) - } - if err := os.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", strconv.Itoa(prometheusPort)); err != nil { - t.T().Fatalf("Failed to set OTEL_EXPORTER_PROMETHEUS_PORT: %v", err) - } + t.Require().NoError(os.Setenv("OTEL_METRICS_EXPORTER", "prometheus"), "failed to set OTEL_METRICS_EXPORTER") + t.Require().NoError(os.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", strconv.Itoa(prometheusPort)), "failed to set OTEL_EXPORTER_PROMETHEUS_PORT") otelRes, err := telemetry.SetupOpenTelemetry(t.ctx) - if err != nil { - t.T().Fatalf("Failed to set up OpenTelemetry: %v", err) - } - if err := InitPerfMetrics(); err != nil { - t.T().Fatalf("Failed to initialize performance metrics: %v", err) - } + t.Require().NoError(err, "failed to set up OpenTelemetry") + t.Require().NoError(InitPerfMetrics(), "failed to initialize performance metrics") t.otelResources = otelRes t.T().Logf("OTel metrics server started on port %v", prometheusPort) SetPerfTestRunInfo("porch-performance-test", t.testOptions.namespace, string(apiVersion), time.Now()) @@ -309,9 +287,7 @@ func (t *PerfTestSuite) SetupSuite() { t.T().Logf(" %d revisions per package", t.testOptions.numRevs) t.T().Logf(" Prometheus metrics: %v", t.enablePrometheus) - if err = t.setupNamespaceAndSecret(); err != nil { - t.T().Fatalf("failed to setup namespace and secret: %v", err) - } + t.Require().NoError(t.setupNamespaceAndSecret(), "failed to setup namespace and secret") t.T().Logf("Created namespace %s and gitea secret", t.testOptions.namespace) t.T().Log("\n=== Cleaning up existing resources from previous runs ===") @@ -643,7 +619,7 @@ func (t *PerfTestSuite) readPackageResources(dir string) map[string]string { } content, readErr := os.ReadFile(path) if readErr != nil { - t.T().Fatalf("ReadFile(%q) failed: %v", path, readErr) + return fmt.Errorf("ReadFile(%q) failed: %w", path, readErr) } relPath, relErr := filepath.Rel(dir, path) if relErr != nil { @@ -652,8 +628,6 @@ func (t *PerfTestSuite) readPackageResources(dir string) map[string]string { resources[relPath] = string(content) return nil }) - if err != nil { - t.T().Fatalf("WalkDir(%s) failed: %v", dir, err) - } + t.Require().NoError(err) return resources } From 9cbb669abccd554ed07d7d738b324621505876c5 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Sun, 2 Aug 2026 16:11:25 +0200 Subject: [PATCH 12/13] Fix from rebase Signed-off-by: Rendre Greyling --- pkg/apiserver/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apiserver/config.go b/pkg/apiserver/config.go index 6eb921f85..75425d9fc 100644 --- a/pkg/apiserver/config.go +++ b/pkg/apiserver/config.go @@ -49,8 +49,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) const NameIndexKey = "metadata.name" From e5dc9704fc7dbed8e958cea0bac05416a19bc122 Mon Sep 17 00:00:00 2001 From: Rendre Greyling Date: Wed, 5 Aug 2026 13:49:15 +0200 Subject: [PATCH 13/13] Address context review Signed-off-by: Rendre Greyling --- internal/telemetry/metrics.go | 13 +++---------- internal/telemetry/metrics_test.go | 7 ------- pkg/util/context/context.go | 8 ++++++++ 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index 67c274dc4..eafe5dbab 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -21,10 +21,10 @@ import ( "github.com/kptdev/porch/api/porch/v1alpha1" "github.com/kptdev/porch/api/porch/v1alpha2" "github.com/kptdev/porch/pkg/repository" + porchcontext "github.com/kptdev/porch/pkg/util/context" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" ) @@ -121,7 +121,7 @@ func RecordRequestCount(ctx context.Context, resource, op, apiVersion string) { klog.Warning("requestsTotal is nil - was InitMetrics() called?") return } - recordRequestCount(resource, op, apiVersion, getK8sUserName(ctx)) + recordRequestCount(resource, op, apiVersion, porchcontext.GetK8sUserName(ctx)) } func RecordControllerRequestCount(resource, op, apiVersion string) { @@ -177,7 +177,7 @@ func RecordExternalRepoRequestCount(ctx context.Context, op string) { metric.WithAttributes( attribute.String("resource", ResourceExternalRepo), attribute.String("op", op), - attribute.String("user", getK8sUserName(ctx)), + attribute.String("user", porchcontext.GetK8sUserName(ctx)), ), ) } @@ -233,10 +233,3 @@ func packageSizeBucketBoundaries() []float64 { copy(buckets[1:], doubled) return buckets } - -func getK8sUserName(ctx context.Context) string { - if user, ok := request.UserFrom(ctx); ok { - return user.GetName() - } - return "" -} diff --git a/internal/telemetry/metrics_test.go b/internal/telemetry/metrics_test.go index 89427c676..06e73bb75 100644 --- a/internal/telemetry/metrics_test.go +++ b/internal/telemetry/metrics_test.go @@ -320,10 +320,3 @@ func TestRecordPackageRevisionResourcesSize_VerboseLogging(t *testing.T) { rm := collectMetricData(t, reader) assert.True(t, hasMetric(rm, "porch_package_size_bytes")) } - -func TestGetK8sUserName(t *testing.T) { - assert.Equal(t, "", getK8sUserName(context.Background())) - - ctx := request.WithUser(context.Background(), &user.DefaultInfo{Name: "alice"}) - assert.Equal(t, "alice", getK8sUserName(ctx)) -} diff --git a/pkg/util/context/context.go b/pkg/util/context/context.go index ee8c857df..26b481482 100644 --- a/pkg/util/context/context.go +++ b/pkg/util/context/context.go @@ -19,6 +19,7 @@ import ( "context" "github.com/google/uuid" + "k8s.io/apiserver/pkg/endpoints/request" ) type porchContextKey string @@ -90,3 +91,10 @@ func LogMetadataFromWithExtras(ctx context.Context, extras ...any) []any { return append(LogMetadataFrom(ctx), extras...) } + +func GetK8sUserName(ctx context.Context) string { + if user, ok := request.UserFrom(ctx); ok { + return user.GetName() + } + return "" +}