diff --git a/cmd/porch/main.go b/cmd/porch/main.go index 9addef8f5..f1705a38e 100644 --- a/cmd/porch/main.go +++ b/cmd/porch/main.go @@ -1,4 +1,4 @@ -// Copyright 2022 The kpt Authors +// Copyright 2022, 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,8 +16,9 @@ package main import ( "os" + "time" - porchotel "github.com/kptdev/porch/internal/otel" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/cmd/server" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/component-base/cli" @@ -34,12 +35,18 @@ func main() { func run() int { log.SetLogger(zap.New(zap.UseDevMode(true))) ctx := genericapiserver.SetupSignalContext() - err := porchotel.SetupOpenTelemetry(ctx) + otelResources, err := telemetry.SetupOpenTelemetry(ctx) if err != nil { genericapiserver.RequestShutdown() klog.Errorf("%v\n", err) return 1 } + defer func() { + if err := otelResources.ShutdownWithTimeout(10 * time.Second); err != nil { + klog.Warningf("failed to gracefully shutdown OpenTelemetry: %v", err) + } + }() + options := server.NewPorchServerOptions(os.Stdout, os.Stderr) cmd := server.NewCommandStartPorchServer(ctx, options) code := cli.Run(cmd) diff --git a/controllers/main.go b/controllers/main.go index 9399e6bd1..e575e4e47 100644 --- a/controllers/main.go +++ b/controllers/main.go @@ -25,6 +25,7 @@ import ( "net/http" "os" "strings" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -41,7 +42,7 @@ import ( porchapi "github.com/kptdev/porch/api/porch/v1alpha1" "github.com/kptdev/porch/controllers/packagevariants/pkg/controllers/packagevariant" "github.com/kptdev/porch/controllers/packagevariantsets/pkg/controllers/packagevariantset" - porchotel "github.com/kptdev/porch/internal/otel" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/controllerrestmapper" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -158,10 +159,15 @@ func run(ctx context.Context) error { } otel.SetLogger(klog.NewKlogr()) - err := porchotel.SetupOpenTelemetry(ctx) + otelResources, err := telemetry.SetupOpenTelemetry(ctx) if err != nil { return fmt.Errorf("error setting up OpenTelemetry: %w", err) } + defer func() { + if shutdownErr := otelResources.ShutdownWithTimeout(10 * time.Second); shutdownErr != nil { + klog.Warningf("failed to gracefully shutdown OpenTelemetry: %v", shutdownErr) + } + }() mgr, err := ctrl.NewManager(cfg, managerOptions) if err != nil { diff --git a/deployments/function-pods/deployment.yaml b/deployments/function-pods/deployment.yaml index 9dcee5ed3..87421edb5 100644 --- a/deployments/function-pods/deployment.yaml +++ b/deployments/function-pods/deployment.yaml @@ -43,6 +43,18 @@ data: volumeMounts: - name: wrapper-server-tools mountPath: /wrapper-server-tools + env: + - name: OTEL_METRICS_EXPORTER + value: prometheus + - name: OTEL_TRACES_EXPORTER + value: none + - name: OTEL_EXPORTER_PROMETHEUS_HOST + value: 0.0.0.0 + - name: OTEL_EXPORTER_PROMETHEUS_PORT + value: "9464" # Default value, showing for visibility + ports: + - containerPort: 9464 + name: metrics volumes: - name: wrapper-server-tools emptyDir: {} @@ -54,6 +66,11 @@ data: - port: 9446 protocol: TCP targetPort: 9446 + name: server + - port: 9464 + protocol: TCP + targetPort: 9464 + name: metrics selector: fn.kpt.dev/image: to-be-replaced type: ClusterIP diff --git a/deployments/metrics-resources/grafana-package-sizes-dashboard.json b/deployments/metrics-resources/grafana-package-sizes-dashboard.json new file mode 100644 index 000000000..68aebf826 --- /dev/null +++ b/deployments/metrics-resources/grafana-package-sizes-dashboard.json @@ -0,0 +1,49 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Total file size, in bytes, of package revisions' resources", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "axisCenteredZero": false, "axisLabel": "", "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 100, "w": 24, "x": 0, "y": 0 }, + "id": 101, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "porch_package_size_bytes_total", + "legendFormat": "{{namespace}}/{{package}}/{{workspace_name}}", + "range": true, + "refId": "total-size-gauge" + } + ], + "title": "PR Resource Sizes", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["porch", "resources"], + "templating": { "list": [] }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Porch File-system Resources", + "uid": "porch-package-resources", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/deployments/metrics-resources/prometheus-config.yaml b/deployments/metrics-resources/prometheus-config.yaml new file mode 100644 index 000000000..0ef2d3f9e --- /dev/null +++ b/deployments/metrics-resources/prometheus-config.yaml @@ -0,0 +1,48 @@ +# Copyright 2026 The kpt Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global: + scrape_native_histograms: true + scrape_interval: 10s + evaluation_interval: 10s + external_labels: + cluster: 'porch' + +scrape_configs: + - job_name: 'porch-server' + scrape_native_histograms: true + scrape_interval: 10s + scrape_timeout: 10s + static_configs: + - targets: ['api.porch-system.svc.cluster.local:9464'] + labels: + service: 'porch' + + - job_name: 'porch-controllers' + scrape_native_histograms: true + scrape_interval: 10s + scrape_timeout: 10s + static_configs: + - targets: ['porch-controllers.porch-system.svc.cluster.local:9464'] + labels: + service: 'porch' + + - job_name: 'function-runner' + scrape_interval: 10s + scrape_timeout: 10s + static_configs: + - targets: ['function-runner.porch-system.svc.cluster.local:9464'] + labels: + service: 'function-runner' + component: 'grpc-server' \ No newline at end of file diff --git a/deployments/metrics/grafana-deployment.yaml b/deployments/metrics/grafana-deployment.yaml new file mode 100644 index 000000000..86e0a85a8 --- /dev/null +++ b/deployments/metrics/grafana-deployment.yaml @@ -0,0 +1,149 @@ +# Copyright 2026 The kpt Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: grafana +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: docker.io/grafana/grafana:latest # kpt-set: ${grafana-image} + ports: + - containerPort: 3000 # kpt-set: ${grafana-container-port} + name: http + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GF_SECURITY_ADMIN_USER + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GF_SECURITY_ADMIN_PASSWORD + - name: GF_USERS_ALLOW_SIGN_UP + value: "false" + - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH + value: "/var/lib/grafana/dashboards/grafana-porch-server-dashboard.json" + - name: GF_DASHBOARDS_MIN_REFRESH_INTERVAL + value: "1s" + - name: GF_SERVER_ENABLE_GZIP + value: "true" + - name: GF_DATABASE_WAL + value: "true" + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + volumeMounts: + - name: grafana-storage + mountPath: /var/lib/grafana + - name: grafana-datasources + mountPath: /etc/grafana/provisioning/datasources + - name: grafana-dashboards-provider + mountPath: /etc/grafana/provisioning/dashboards + - name: grafana-dashboards + mountPath: /var/lib/grafana/dashboards + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "1" + volumes: + - name: grafana-storage + emptyDir: {} + - name: grafana-datasources + configMap: + name: grafana-datasources + - name: grafana-dashboards-provider + configMap: + name: grafana-dashboards-provider + - name: grafana-dashboards + configMap: + name: grafana-dashboards +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-provider + namespace: porch-monitoring # kpt-set: ${namespace} +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: porch-monitoring # kpt-set: ${namespace} +data: + prometheus.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + uid: prometheus + url: http://prometheus:9090 + isDefault: true + editable: true +--- +apiVersion: v1 +data: + GF_SECURITY_ADMIN_USER: cG9yY2g= # kpt-set: ${grafana-user} + GF_SECURITY_ADMIN_PASSWORD: cG9yY2g= # kpt-set: ${grafana-pw} +kind: Secret +metadata: + name: grafana-admin-creds + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: grafana +type: Opaque diff --git a/deployments/metrics/prometheus-deployment.yaml b/deployments/metrics/prometheus-deployment.yaml new file mode 100644 index 000000000..187540380 --- /dev/null +++ b/deployments/metrics/prometheus-deployment.yaml @@ -0,0 +1,117 @@ +# Copyright 2026 The kpt Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: prometheus + containers: + - name: prometheus + image: docker.io/prom/prometheus:latest # kpt-set: ${prometheus-image} + args: + - '--config.file=/etc/prometheus/prometheus-config.yaml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--storage.tsdb.retention.time=6h' + - '--query.timeout=1m' + ports: + - containerPort: 9090 # kpt-set: ${prometheus-container-port} + name: http + volumeMounts: + - name: prometheus-config + mountPath: /etc/prometheus + - name: prometheus-storage + mountPath: /prometheus + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + volumes: + - name: prometheus-config + configMap: + name: prometheus-config + - name: prometheus-storage + emptyDir: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: porch-monitoring # kpt-set: ${namespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: porch-monitoring # kpt-set: ${namespace} +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: porch-monitoring # kpt-set: ${namespace} + labels: + app: prometheus +spec: + type: ClusterIP + ports: + - port: 9090 # kpt-set: ${prometheus-container-port} + targetPort: 9090 # kpt-set: ${prometheus-container-port} + name: http + selector: + app: prometheus diff --git a/deployments/porch/2-function-runner.yaml b/deployments/porch/2-function-runner.yaml index 3a727625b..4b9a813ee 100644 --- a/deployments/porch/2-function-runner.yaml +++ b/deployments/porch/2-function-runner.yaml @@ -100,6 +100,11 @@ spec: - port: 9445 protocol: TCP targetPort: 9445 + name: grpc + - port: 9464 + protocol: TCP + targetPort: 9464 + name: metrics --- apiVersion: v1 diff --git a/deployments/porch/3-porch-server.yaml b/deployments/porch/3-porch-server.yaml index 2428b84e2..33e123e78 100644 --- a/deployments/porch/3-porch-server.yaml +++ b/deployments/porch/3-porch-server.yaml @@ -94,7 +94,20 @@ spec: - --repo-operation-retry-attempts=3 - --max-request-body-size=6291456 # Keep this in sync with function-runner's corresponding argument - --cache-type=db + ports: + - containerPort: 9464 + name: metrics + protocol: TCP #adding livenessProbes and readinessProbes for porch server + startupProbe: + httpGet: + path: /healthz + port: 4443 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 30 + timeoutSeconds: 5 livenessProbe: httpGet: path: /healthz @@ -131,5 +144,9 @@ spec: protocol: TCP targetPort: 8443 name: webhooks + - port: 9464 + protocol: TCP + targetPort: 9464 + name: metrics selector: app: porch-server diff --git a/deployments/porch/9-controllers.yaml b/deployments/porch/9-controllers.yaml index 0fe3181d7..7cb20f22d 100644 --- a/deployments/porch/9-controllers.yaml +++ b/deployments/porch/9-controllers.yaml @@ -80,4 +80,18 @@ spec: initialDelaySeconds: 5 periodSeconds: 5 successThreshold: 1 - timeoutSeconds: 3 + timeoutSeconds: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: porch-controllers + namespace: porch-system +spec: + ports: + - port: 9464 + protocol: TCP + targetPort: 9464 + name: metrics + selector: + k8s-app: porch-controllers diff --git a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md index 8b10fe3d6..d9efd651b 100644 --- a/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md +++ b/docs/content/en/docs/6_configuration_and_deployments/configurations/opentelemetry.md @@ -410,6 +410,43 @@ env: This allows routing different telemetry signals to specialized backends. + +## Available Metrics + +Porch records the following metrics via OpenTelemetry: + +### Package Size Metrics + +| Metric Name | Type | Unit | Description | +|----------------------------------|-----------|-------|-------------| +| `porch_package_size_bytes` | Histogram | Bytes | File size of a package's resources expressed as a histogram | +| `porch_package_size_bytes_total` | Gauge | Bytes | Total file size of a package's resources | + +Package size metrics are recorded with the following attributes from the relevant package: + +| Attribute | Description | +|------------------|-------------| +| `namespace` | Kubernetes namespace of the package revision | +| `repository` | Name of the repository containing the package | +| `package` | Path and name of the package | +| `workspace_name` | WorkspaceName of the package revision - short, unique description of the changes | + +These metrics are recorded as part of every flow that updates package revision resources: +- Create package revision +- Delete package revision +- Discover/sync package revisions from a registered repository +- Delete package revisions on unregistering a repository +- Direct update of PackageRevisionResources (e.g. `rpkg push`) + +**Prometheus metric names:** + +When using the Prometheus exporter, these are made available under the metric names: +- `porch_package_size_bytes_bucket` +- `porch_package_size_bytes_count` +- `porch_package_size_bytes_sum` +- `porch_package_size_bytes_total` + + ## Troubleshooting ### Verify Metrics Endpoint diff --git a/func/server/server.go b/func/server/server.go index da2fff0e5..534172a3e 100644 --- a/func/server/server.go +++ b/func/server/server.go @@ -1,4 +1,4 @@ -// Copyright 2022-2025 The kpt Authors +// Copyright 2022-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import ( pb "github.com/kptdev/porch/func/evaluator" "github.com/kptdev/porch/func/healthchecker" "github.com/kptdev/porch/func/internal" - porchotel "github.com/kptdev/porch/internal/otel" + "github.com/kptdev/porch/internal/telemetry" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" @@ -105,12 +105,17 @@ func run(o *options) error { lis.Close() }() - err = porchotel.SetupOpenTelemetry(ctx) + otelResources, err := telemetry.SetupOpenTelemetry(ctx) if err != nil { contextsignal.RequestShutdown() klog.Errorf("%v\n", err) return err } + defer func() { + if err := otelResources.ShutdownWithTimeout(10 * time.Second); err != nil { + klog.Warningf("failed to gracefully shutdown OpenTelemetry: %v", err) + } + }() availableRuntimes := map[string]struct{}{ execRuntime: {}, diff --git a/func/wrapper-server/main.go b/func/wrapper-server/main.go index 0030af3c2..a2cb2b093 100644 --- a/func/wrapper-server/main.go +++ b/func/wrapper-server/main.go @@ -1,4 +1,4 @@ -// Copyright 2022 The kpt Authors +// Copyright 2022, 2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,11 +25,12 @@ import ( "os" "os/exec" "strconv" + "time" "github.com/kptdev/krm-functions-sdk/go/fn" pb "github.com/kptdev/porch/func/evaluator" "github.com/kptdev/porch/func/healthchecker" - porchotel "github.com/kptdev/porch/internal/otel" + "github.com/kptdev/porch/internal/telemetry" "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" @@ -79,12 +80,17 @@ type options struct { func (o *options) run() error { ctx := contextsignal.SetupSignalContext() - err := porchotel.SetupOpenTelemetry(ctx) + otelResources, err := telemetry.SetupOpenTelemetry(ctx) if err != nil { contextsignal.RequestShutdown() klog.Errorf("%v\n", err) return err } + defer func() { + if err := otelResources.ShutdownWithTimeout(10 * time.Second); err != nil { + klog.Warningf("failed to gracefully shutdown OpenTelemetry: %v", err) + } + }() klog.Info("OpenTelemetry initialized") address := fmt.Sprintf(":%d", o.port) lis, err := net.Listen("tcp", address) diff --git a/go.mod b/go.mod index 96f03bc9e..33172a845 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,8 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 go.opentelemetry.io/contrib/propagators/autoprop v0.63.0 go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/prometheus v0.65.0 + go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 @@ -93,7 +95,6 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.65.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.19.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 // indirect @@ -216,7 +217,6 @@ require ( go.mongodb.org/mongo-driver v1.17.6 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect diff --git a/internal/otel/otel.go b/internal/otel/otel.go deleted file mode 100644 index 48e17a856..000000000 --- a/internal/otel/otel.go +++ /dev/null @@ -1,80 +0,0 @@ -package porch - -import ( - "context" - "fmt" - "net/http" - "time" - - "go.opentelemetry.io/contrib/bridges/prometheus" - "go.opentelemetry.io/contrib/exporters/autoexport" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/contrib/propagators/autoprop" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/trace" - "k8s.io/klog/v2" - controllerruntimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics" -) - -// Sets up OpenTelemetry with parameters -// from environment variables based on the -// opentelemetry.io/contrib/exporters/autoexport" -func SetupOpenTelemetry(ctx context.Context) error { - setupTiming := time.Now() - err := setupTracing(ctx) - if err != nil { - return err - } - err = setupMetrics(ctx) - if err != nil { - return err - } - http.DefaultTransport = otelhttp.NewTransport(http.DefaultTransport) - http.DefaultClient.Transport = http.DefaultTransport - klog.Infof("OpenTelemetry initialized in %s", time.Since(setupTiming)) - return nil - -} - -func setupTracing(ctx context.Context) error { - exp, err := autoexport.NewSpanExporter(ctx) - if err != nil { - return fmt.Errorf("failed to create span exporter: %w", err) - } - tp := trace.NewTracerProvider(trace.WithBatcher(exp)) - go func() { - <-ctx.Done() - if err := tp.Shutdown(context.Background()); err != nil { - panic(err) - } - }() - otel.SetTracerProvider(tp) - otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) - - return nil -} - -func setupMetrics(ctx context.Context) error { - autoexport.WithFallbackMetricProducer(func(ctx context.Context) (metric.Producer, error) { - return prometheus.NewMetricProducer( - prometheus.WithGatherer(controllerruntimemetrics.Registry), - ), nil - }) - - mr, err := autoexport.NewMetricReader(ctx) - if err != nil { - return fmt.Errorf("failed to create metric reader: %w", err) - } - go func() { - <-ctx.Done() - if err := mr.Shutdown(context.Background()); err != nil { - panic(err) - } - }() - - mp := metric.NewMeterProvider(metric.WithReader(mr)) - otel.SetMeterProvider(mp) - - return nil -} diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go new file mode 100644 index 000000000..91ac2c2d3 --- /dev/null +++ b/internal/telemetry/metrics.go @@ -0,0 +1,94 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + + "github.com/kptdev/porch/pkg/repository" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "k8s.io/klog/v2" +) + +const meterName = "github.com/kptdev/porch" + +var ( + prResourceSizeHistogram metric.Int64Histogram + prResourceSizeGauge metric.Int64Gauge +) + +func InitMetrics() (err error) { + m := otel.Meter(meterName) + + prResourceSizeHistogram, err = m.Int64Histogram( + "porch_package_size_bytes", + metric.WithUnit("By"), + metric.WithDescription("Distribution of package revision resources' file size, in bytes"), + metric.WithExplicitBucketBoundaries(0, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824), + ) + if err != nil { + klog.Errorf("failed to create porch_package_size_bytes histogram: %v", err) + return + } + + prResourceSizeGauge, err = m.Int64Gauge( + "porch_package_size_bytes_total", + metric.WithUnit("By"), + metric.WithDescription("Total file size, in bytes, of a package revision's resources"), + ) + if err != nil { + klog.Errorf("failed to create porch_package_size_bytes gauge: %v", err) + return + } + + return nil +} + +// Porch server and function runner metric recording functions +func RecordPackageRevisionResourcesSize(ctx context.Context, prKey repository.PackageRevisionKey, resourcesSize int64) { + prPath := func() string { + if prKey.PKey().Path != "" { + return prKey.PKey().Path + "/" + } + return "" + }() + attributes := attribute.NewSet( + attribute.String("namespace", prKey.RKey().Namespace), + attribute.String("repository", prKey.RKey().Name), + attribute.String("package", prPath+prKey.PKey().Package), + attribute.String("workspace_name", prKey.WorkspaceName), + ) + + if prResourceSizeHistogram == nil { + klog.Warning("prResourceSizeHistogram is nil - was InitMetrics() called?") + return + } + + if klog.V(3).Enabled() { + klog.Infof( + "Recording package resources size %dB for package revision with attributes %v", + resourcesSize, attributes.MarshalLog()) + } + + prResourceSizeHistogram.Record(ctx, resourcesSize, metric.WithAttributeSet(attributes)) + + if prResourceSizeGauge == nil { + klog.Warning("prResourceSizeGauge is nil - was InitMetrics() called?") + return + } + prResourceSizeGauge.Record(ctx, resourcesSize, metric.WithAttributeSet(attributes)) +} diff --git a/internal/telemetry/metrics_test.go b/internal/telemetry/metrics_test.go new file mode 100644 index 000000000..923fee63e --- /dev/null +++ b/internal/telemetry/metrics_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + "testing" + + "github.com/kptdev/porch/pkg/repository" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +type fakePackageRevision struct { + repository.PackageRevision + key repository.PackageRevisionKey + namespace string +} + +func (f *fakePackageRevision) KubeObjectNamespace() string { return f.namespace } +func (f *fakePackageRevision) Key() repository.PackageRevisionKey { return f.key } + +// Remaining interface methods are not called by RecordPackageRevisionResourcesSize, +// so they can panic if invoked unexpectedly. + +func TestRecordPackageRevisionResourcesSize_NilInstruments(t *testing.T) { + require.NoError(t, InitMetrics()) + histogramBefore := prResourceSizeHistogram + prResourceSizeHistogram = nil + defer func() { prResourceSizeHistogram = histogramBefore }() + + fake := + repository.PackageRevisionKey{ + PkgKey: repository.PackageKey{RepoKey: repository.RepositoryKey{Namespace: "ns"}}, + WorkspaceName: "ws", + Revision: 1, + } + // Should return early without panic + assert.NotPanics(t, func() { RecordPackageRevisionResourcesSize(context.Background(), fake, 1024) }) + + prResourceSizeHistogram = histogramBefore + gaugeBefore := prResourceSizeGauge + prResourceSizeGauge = nil + defer func() { prResourceSizeGauge = gaugeBefore }() + // Should return early without panic + assert.NotPanics(t, func() { RecordPackageRevisionResourcesSize(context.Background(), fake, 1024) }) +} + +func TestRecordPackageRevisionResourcesSize_RecordsMetrics(t *testing.T) { + previousMp := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(mp) + defer func() { + otel.SetMeterProvider(previousMp) + mp.Shutdown(context.Background()) + }() + + require.NoError(t, InitMetrics()) + + fake := + repository.PackageRevisionKey{ + PkgKey: repository.PackageKey{RepoKey: repository.RepositoryKey{Namespace: "test-ns"}}, + WorkspaceName: "ws", + Revision: 1, + } + + RecordPackageRevisionResourcesSize(context.Background(), fake, 4096) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + + var foundHistogram, foundGauge bool + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == "porch_package_size_bytes" { + foundHistogram = true + } + if m.Name == "porch_package_size_bytes_total" { + foundGauge = true + } + } + } + assert.True(t, foundHistogram, "expected porch_package_size_bytes histogram to be recorded") + assert.True(t, foundGauge, "expected porch_package_size_bytes_total gauge to be recorded") +} diff --git a/internal/telemetry/otel.go b/internal/telemetry/otel.go new file mode 100644 index 000000000..c13bfac8e --- /dev/null +++ b/internal/telemetry/otel.go @@ -0,0 +1,235 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry + +import ( + "context" + "fmt" + "net/http" + "os" + "strconv" + "time" + + prombridge "go.opentelemetry.io/contrib/bridges/prometheus" + "go.opentelemetry.io/contrib/exporters/autoexport" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/contrib/propagators/autoprop" + "go.opentelemetry.io/otel" + otelprometheus "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/trace" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/klog/v2" + controllerruntimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + otelHostEnv = "OTEL_EXPORTER_PROMETHEUS_HOST" + otelHostDefault = "0.0.0.0" + otelPortEnv = "OTEL_EXPORTER_PROMETHEUS_PORT" + otelPortDefault = "9464" +) + +// OTelResources holds all OpenTelemetry resources that need lifecycle management. +// Use Shutdown() to cleanly release all resources. +type OTelResources struct { + metricsServer *http.Server + metricsPort int + meterProvider *sdkmetric.MeterProvider + tracerProvider *trace.TracerProvider +} + +// Shutdown gracefully shuts down all OpenTelemetry resources. +func (r *OTelResources) Shutdown(ctx context.Context) error { + shutdownTiming := time.Now() + var errs []error + if r.metricsServer != nil { + if err := r.metricsServer.Shutdown(ctx); err != nil { + errs = append(errs, fmt.Errorf("metrics server shutdown: %w", err)) + } + } + if r.meterProvider != nil { + if err := r.meterProvider.Shutdown(ctx); err != nil { + errs = append(errs, fmt.Errorf("meter provider shutdown: %w", err)) + } + } + if r.tracerProvider != nil { + if err := r.tracerProvider.Shutdown(ctx); err != nil { + errs = append(errs, fmt.Errorf("tracer provider shutdown: %w", err)) + } + } + if len(errs) > 0 { + return fmt.Errorf("otel shutdown errors: %v", errs) + } + klog.Infof("OpenTelemetry shut down in %s", time.Since(shutdownTiming)) + return nil +} + +// ShutdownWithTimeout is a convenience wrapper around Shutdown with a timeout. +func (r *OTelResources) ShutdownWithTimeout(timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return r.Shutdown(ctx) +} + +// Flush forces a flush of the meter provider, useful in tests. +func (r *OTelResources) Flush() error { + if r.meterProvider != nil { + return r.meterProvider.ForceFlush(context.Background()) + } + return nil +} + +// SetupOpenTelemetry is the single entry point for all OpenTelemetry setup. +// It configures tracing, metrics (including the Prometheus HTTP server if +// OTEL_EXPORTER_PROMETHEUS_HOST and OTEL_EXPORTER_PROMETHEUS_PORT are set), +// and initializes all Porch metric instruments. Returns OTelResources +// for lifecycle management. +func SetupOpenTelemetry(ctx context.Context) (*OTelResources, error) { + setupTiming := time.Now() + res := &OTelResources{} + + // Setup tracing + if err := setupTracing(ctx, res); err != nil { + return nil, err + } + + // Setup metrics provider + if err := setupMetrics(ctx, res); err != nil { + return nil, err + } + + // Initialize all Porch metric instruments + if err := InitMetrics(); err != nil { + return nil, fmt.Errorf("failed to initialize Porch metrics: %w", err) + } + + // Start the Prometheus metrics HTTP server if port is configured + if err := startMetricsServerIfConfigured(res); err != nil { + return nil, err + } + + http.DefaultTransport = otelhttp.NewTransport(http.DefaultTransport) + http.DefaultClient.Transport = http.DefaultTransport + klog.Infof("OpenTelemetry initialized in %s", time.Since(setupTiming)) + return res, nil +} + +func setupTracing(ctx context.Context, res *OTelResources) error { + exp, err := autoexport.NewSpanExporter(ctx) + if err != nil { + return fmt.Errorf("failed to create span exporter: %w", err) + } + tp := trace.NewTracerProvider(trace.WithBatcher(exp)) + res.tracerProvider = tp + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(autoprop.NewTextMapPropagator()) + return nil +} + +func setupMetrics(ctx context.Context, res *OTelResources) error { + exporter := os.Getenv("OTEL_METRICS_EXPORTER") + + autoexport.WithFallbackMetricProducer(func(ctx context.Context) (sdkmetric.Producer, error) { + return prombridge.NewMetricProducer( + prombridge.WithGatherer(controllerruntimemetrics.Registry), + ), nil + }) + + readers := []sdkmetric.Option{} + if exporter == "prometheus" { + if os.Getenv(otelHostEnv) == "" { + if err := os.Setenv(otelHostEnv, otelHostDefault); err != nil { + return err + } + } + if os.Getenv(otelPortEnv) == "" { + if err := os.Setenv(otelPortEnv, otelPortDefault); err != nil { + return err + } + } + + // Only create the Prometheus exporter when we intend to expose a scrape + // endpoint, to avoid writing OTel metrics into the default Prometheus + // registry when pushing via OTLP. + promExp, err := otelprometheus.New( + otelprometheus.WithRegisterer(prometheus.DefaultRegisterer), + ) + if err != nil { + return fmt.Errorf("failed to create prometheus exporter: %w", err) + } + readers = append(readers, sdkmetric.WithReader(promExp)) + } else { + autoMr, err := autoexport.NewMetricReader(ctx) + if err != nil { + return fmt.Errorf("failed to create metric reader: %w", err) + } + readers = append(readers, sdkmetric.WithReader(autoMr)) + } + + mp := sdkmetric.NewMeterProvider(readers...) + res.meterProvider = mp + otel.SetMeterProvider(mp) + + return nil +} + +func startMetricsServerIfConfigured(res *OTelResources) error { + hostStr := os.Getenv(otelHostEnv) + if hostStr == "" { + return nil + } + portStr := os.Getenv(otelPortEnv) + if portStr == "" { + return nil + } + port, err := strconv.Atoi(portStr) + if err != nil { + return fmt.Errorf("invalid %s value %q: %w", otelPortEnv, portStr, err) + } + if port <= 0 { + return nil + } + + gatherers := prometheus.Gatherers{ + prometheus.DefaultGatherer, + controllerruntimemetrics.Registry, + } + handler := promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{ + ErrorHandling: promhttp.ContinueOnError, + }) + + mux := http.NewServeMux() + mux.Handle("/metrics", handler) + + srv := &http.Server{ + Addr: fmt.Sprintf("%s:%d", hostStr, port), + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } + res.metricsServer = srv + res.metricsPort = port + + go func() { + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + klog.Errorf("OTel metrics server error: %v", err) + } + }() + klog.Infof("OTel metrics server started on port %d", port) + + return nil +} diff --git a/internal/otel/otel_test.go b/internal/telemetry/otel_test.go similarity index 50% rename from internal/otel/otel_test.go rename to internal/telemetry/otel_test.go index 6912be798..1e47a9972 100644 --- a/internal/otel/otel_test.go +++ b/internal/telemetry/otel_test.go @@ -1,4 +1,18 @@ -package porch +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetry import ( "context" @@ -8,6 +22,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,24 +34,80 @@ import ( otlptraces "go.opentelemetry.io/proto/otlp/collector/trace/v1" ) +const ( + ENV_OTEL_METRICS_EXPORTER = "OTEL_METRICS_EXPORTER" + METRICS_EXPORTER_PROMETHEUS = "prometheus" + METRICS_EXPORTER_OTLP = "otlp" + + ENV_OTEL_TRACES_EXPORTER = "OTEL_TRACES_EXPORTER" + DEFAULT_OTEL_TRACES_EXPORTER = "none" + + ENV_OTEL_EXPORTER_PROMETHEUS_HOST = "OTEL_EXPORTER_PROMETHEUS_HOST" + ENV_OTEL_EXPORTER_PROMETHEUS_PORT = "OTEL_EXPORTER_PROMETHEUS_PORT" + ENV_OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT" + ENV_OTEL_EXPORTER_OTLP_PROTOCOL = "OTEL_EXPORTER_OTLP_PROTOCOL" +) + +func TestPrometheusHTTPServer(t *testing.T) { + // Find a free port + lis, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := lis.Addr().(*net.TCPAddr).Port + lis.Close() + + t.Setenv(ENV_OTEL_METRICS_EXPORTER, METRICS_EXPORTER_PROMETHEUS) + t.Setenv(ENV_OTEL_TRACES_EXPORTER, DEFAULT_OTEL_TRACES_EXPORTER) + t.Setenv(ENV_OTEL_EXPORTER_PROMETHEUS_HOST, "0.0.0.0") + t.Setenv(ENV_OTEL_EXPORTER_PROMETHEUS_PORT, fmt.Sprintf("%d", port)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + res, err := SetupOpenTelemetry(ctx) + require.NoError(t, err) + defer res.ShutdownWithTimeout(5 * time.Second) + + // Verify the HTTP server is serving metrics + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/metrics", port)) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(body), "target_info") +} + +func TestPrometheusHTTPServerInvalidPort(t *testing.T) { + t.Setenv(ENV_OTEL_METRICS_EXPORTER, METRICS_EXPORTER_PROMETHEUS) + t.Setenv(ENV_OTEL_TRACES_EXPORTER, DEFAULT_OTEL_TRACES_EXPORTER) + t.Setenv(ENV_OTEL_EXPORTER_PROMETHEUS_HOST, "0.0.0.0") + t.Setenv(ENV_OTEL_EXPORTER_PROMETHEUS_PORT, "not-a-number") + + ctx := context.Background() + _, err := SetupOpenTelemetry(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid") +} + func TestOtelMetricsPushHTTP(t *testing.T) { requestWaitChannel := make(chan struct{}) ts := httptest.NewServer(&mockHTTPMetricsServer{t: t, ch: requestWaitChannel}) defer ts.Close() - t.Setenv("OTEL_METRICS_EXPORTER", "otlp") - t.Setenv("OTEL_TRACES_EXPORTER", "none") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", ts.URL) - t.Setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + t.Setenv(ENV_OTEL_METRICS_EXPORTER, METRICS_EXPORTER_OTLP) + t.Setenv(ENV_OTEL_TRACES_EXPORTER, DEFAULT_OTEL_TRACES_EXPORTER) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_ENDPOINT, ts.URL) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_PROTOCOL, "http/protobuf") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := SetupOpenTelemetry(ctx) + res, err := SetupOpenTelemetry(ctx) require.NoError(t, err) - cancel() + // Shutdown flushes the periodic reader, which triggers the export + require.NoError(t, res.ShutdownWithTimeout(5*time.Second)) <-requestWaitChannel } @@ -46,15 +117,15 @@ func TestOtelTracesPushHTTP(t *testing.T) { ts := httptest.NewServer(&mockHTTPTraceServer{t: t, ch: requestWaitChannel}) defer ts.Close() - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("OTEL_METRICS_EXPORTER", "none") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", ts.URL) - t.Setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + t.Setenv(ENV_OTEL_TRACES_EXPORTER, METRICS_EXPORTER_OTLP) + t.Setenv(ENV_OTEL_METRICS_EXPORTER, DEFAULT_OTEL_TRACES_EXPORTER) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_ENDPOINT, ts.URL) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_PROTOCOL, "http/protobuf") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := SetupOpenTelemetry(ctx) + res, err := SetupOpenTelemetry(ctx) require.NoError(t, err) // Create a span to trigger trace export @@ -62,36 +133,26 @@ func TestOtelTracesPushHTTP(t *testing.T) { _, span := tracer.Start(ctx, "test-span") span.End() + // Shutdown flushes the batch span processor + require.NoError(t, res.ShutdownWithTimeout(5*time.Second)) <-requestWaitChannel } func TestSetupOpenTelemetryPrometheusEndpoint(t *testing.T) { - // Find available port - listener, err := net.Listen("tcp", ":0") - require.NoError(t, err) - port := listener.Addr().(*net.TCPAddr).Port - listener.Close() - - t.Setenv("OTEL_METRICS_EXPORTER", "prometheus") - t.Setenv("OTEL_EXPORTER_PROMETHEUS_HOST", "localhost") - t.Setenv("OTEL_EXPORTER_PROMETHEUS_PORT", fmt.Sprintf("%d", port)) + t.Setenv(ENV_OTEL_METRICS_EXPORTER, METRICS_EXPORTER_PROMETHEUS) + t.Setenv(ENV_OTEL_TRACES_EXPORTER, DEFAULT_OTEL_TRACES_EXPORTER) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err = SetupOpenTelemetry(ctx) - require.NoError(t, err) - - // Make request to the Prometheus metrics endpoint - resp, err := http.Get(fmt.Sprintf("http://localhost:%d/metrics", port)) + res, err := SetupOpenTelemetry(ctx) require.NoError(t, err) - defer resp.Body.Close() + defer res.ShutdownWithTimeout(5 * time.Second) - body, err := io.ReadAll(resp.Body) + // Verify that metrics are accessible via the OTel meter provider + meter := otel.Meter("test") + counter, err := meter.Float64Counter("test_counter") require.NoError(t, err) - - metricsText := string(body) - // Verify at least one metric line exists (non-comment, non-empty) - assert.Regexp(t, `(?m)^([a-zA-Z_][a-zA-Z0-9_]*)`, metricsText) + counter.Add(ctx, 1) } func TestOtelMetricsPushGRPC(t *testing.T) { @@ -111,18 +172,18 @@ func TestOtelMetricsPushGRPC(t *testing.T) { }() defer s.Stop() - t.Setenv("OTEL_METRICS_EXPORTER", "otlp") - t.Setenv("OTEL_TRACES_EXPORTER", "none") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", fmt.Sprintf("http://localhost:%d", lis.Addr().(*net.TCPAddr).Port)) - t.Setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc") + t.Setenv(ENV_OTEL_METRICS_EXPORTER, METRICS_EXPORTER_OTLP) + t.Setenv(ENV_OTEL_TRACES_EXPORTER, DEFAULT_OTEL_TRACES_EXPORTER) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_ENDPOINT, fmt.Sprintf("http://localhost:%d", lis.Addr().(*net.TCPAddr).Port)) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_PROTOCOL, "grpc") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err = SetupOpenTelemetry(ctx) + res, err := SetupOpenTelemetry(ctx) require.NoError(t, err) - cancel() + require.NoError(t, res.ShutdownWithTimeout(5*time.Second)) <-requestWaitChannel } @@ -143,15 +204,15 @@ func TestOtelTracesPushGRPC(t *testing.T) { }() defer s.Stop() - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("OTEL_METRICS_EXPORTER", "none") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", fmt.Sprintf("http://localhost:%d", lis.Addr().(*net.TCPAddr).Port)) - t.Setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc") + t.Setenv(ENV_OTEL_TRACES_EXPORTER, METRICS_EXPORTER_OTLP) + t.Setenv(ENV_OTEL_METRICS_EXPORTER, "none") + t.Setenv(ENV_OTEL_EXPORTER_OTLP_ENDPOINT, fmt.Sprintf("http://localhost:%d", lis.Addr().(*net.TCPAddr).Port)) + t.Setenv(ENV_OTEL_EXPORTER_OTLP_PROTOCOL, "grpc") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err = SetupOpenTelemetry(ctx) + res, err := SetupOpenTelemetry(ctx) require.NoError(t, err) // Create a span to trigger trace export @@ -159,7 +220,7 @@ func TestOtelTracesPushGRPC(t *testing.T) { _, span := tracer.Start(ctx, "test-span") span.End() - cancel() + require.NoError(t, res.ShutdownWithTimeout(5*time.Second)) <-requestWaitChannel } diff --git a/pkg/cache/dbcache/dbpackagerevisionresourcessql.go b/pkg/cache/dbcache/dbpackagerevisionresourcessql.go index 9becaf2a9..4fd894fd5 100644 --- a/pkg/cache/dbcache/dbpackagerevisionresourcessql.go +++ b/pkg/cache/dbcache/dbpackagerevisionresourcessql.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The kpt Authors +// Copyright 2024-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/repository" "go.opentelemetry.io/otel/trace" "k8s.io/klog/v2" @@ -142,10 +143,10 @@ func pkgRevResourcesDeleteFromDB(ctx context.Context, prk repository.PackageRevi if err == nil { klog.V(5).Infof("pkgRevResourcesDeleteFromDB: deleted package revision resources for %+v", prk) + telemetry.RecordPackageRevisionResourcesSize(ctx, prk, 0) } else { klog.Warningf("pkgRevResourcesDeleteFromDB: deletion of package revision resources for %+v failed: %q", prk, err) } - return err } diff --git a/pkg/cache/dbcache/dbrepository.go b/pkg/cache/dbcache/dbrepository.go index 445b929ff..3ac56ea52 100644 --- a/pkg/cache/dbcache/dbrepository.go +++ b/pkg/cache/dbcache/dbrepository.go @@ -24,6 +24,7 @@ import ( kptfilev1 "github.com/kptdev/kpt/pkg/api/kptfile/v1" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" cachetypes "github.com/kptdev/porch/pkg/cache/types" "github.com/kptdev/porch/pkg/externalrepo" externalrepotypes "github.com/kptdev/porch/pkg/externalrepo/types" @@ -378,6 +379,11 @@ func (r *dbRepository) ClosePackageRevisionDraft(ctx context.Context, prd reposi } pr, err := r.savePackageRevisionDraft(ctx, prd, version) + if err != nil { + return nil, err + } + + telemetry.RecordPackageRevisionResourcesSize(ctx, pr.Key(), pr.resourcesSizeBytes) return repository.PackageRevision(pr), err } diff --git a/pkg/cache/dbcache/dbreposync.go b/pkg/cache/dbcache/dbreposync.go index 427943b3e..b956360a0 100644 --- a/pkg/cache/dbcache/dbreposync.go +++ b/pkg/cache/dbcache/dbreposync.go @@ -1,4 +1,4 @@ -// Copyright 2025 The kpt Authors +// Copyright 2025-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import ( porchapi "github.com/kptdev/porch/api/porch/v1alpha1" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" + "github.com/kptdev/porch/internal/telemetry" "github.com/kptdev/porch/pkg/cache/sync" cachetypes "github.com/kptdev/porch/pkg/cache/types" "github.com/kptdev/porch/pkg/repository" @@ -281,6 +282,8 @@ func (s *repositorySync) cacheExternalPRs(ctx context.Context, externalPrMap map klog.Errorf("repositorySync %+v: failed to save external package revision %+v to database", s.repo.Key(), extPRKey) return err } + + telemetry.RecordPackageRevisionResourcesSize(ctx, dbPR.Key(), dbPR.resourcesSizeBytes) } return nil diff --git a/pkg/cli/commands/rpkg/approve/command_test.go b/pkg/cli/commands/rpkg/approve/command_test.go index 7022d07f6..5d5aabac5 100644 --- a/pkg/cli/commands/rpkg/approve/command_test.go +++ b/pkg/cli/commands/rpkg/approve/command_test.go @@ -141,7 +141,7 @@ func TestCmd(t *testing.T) { "Approve deletion-proposed package": { output: pkgRevName + " approved\n", fakeclient: fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ - //fake subresourceupdate + // fake SubResourceUpdate SubResourceUpdate: func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { return nil }, diff --git a/pkg/cli/commands/rpkg/docs/docs.go b/pkg/cli/commands/rpkg/docs/docs.go index 979b68ccb..681d9d5e2 100644 --- a/pkg/cli/commands/rpkg/docs/docs.go +++ b/pkg/cli/commands/rpkg/docs/docs.go @@ -245,7 +245,7 @@ var PullLong = ` Args: K8S_PACKAGE_REV_NAME: - The kubernetes name of a an existing package revision in a repository. + The kubernetes name of an existing package revision in a repository. DIR: A local directory where the package manifests will be written. @@ -263,7 +263,7 @@ var PushLong = ` Args: K8S_PACKAGE_REV_NAME: - The kubernetes name of a an existing package revision in a repository. + The kubernetes name of an existing package revision in a repository. DIR: A local directory with the new manifest. If the manifests have be read from stdin, use '-' in place of DIR. diff --git a/scripts/deploy-monitoring.sh b/scripts/deploy-monitoring.sh new file mode 100755 index 000000000..83287754a --- /dev/null +++ b/scripts/deploy-monitoring.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# Copyright 2026 The kpt Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +METRICS_DIR="${SCRIPT_DIR}/../deployments/metrics" +DOT_ENV_PATH="${SCRIPT_DIR}/../.env" +PORT_FORWARD_DIR="$(mktemp --directory --suffix "_porch-monitoring-pf.pid.d")" + +if [[ -f "$DOT_ENV_PATH" ]]; then + source "$DOT_ENV_PATH" +fi + +# Configuration +NAMESPACE="${NAMESPACE:-porch-monitoring}" +PROMETHEUS_LOCAL_PORT="${PROMETHEUS_LOCAL_PORT:-9092}" +PROMETHEUS_CONTAINER_PORT="${PROMETHEUS_CONTAINER_PORT:-9090}" +PROMETHEUS_NODEPORT="${PROMETHEUS_NODEPORT:-30091}" +GRAFANA_LOCAL_PORT="${GRAFANA_LOCAL_PORT:-3001}" +GRAFANA_CONTAINER_PORT="${GRAFANA_CONTAINER_PORT:-3000}" +GRAFANA_NODEPORT="${GRAFANA_NODEPORT:-30301}" + +GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-porch}" +GRAFANA_ADMIN_PW="${GRAFANA_ADMIN_PW:-}" +[[ -z $GRAFANA_ADMIN_PW ]] && GRAFANA_ADMIN_PW="$(date +%s | shasum -a 256 | base64 | head -c 15)" + +DOCKERHUB_MIRROR="${DOCKERHUB_MIRROR:-docker.io}" +KRM_FN_REGISTRY_URL="${KRM_FN_REGISTRY_URL:-ghcr.io/kptdev/krm-functions-catalog}" +PROMETHEUS_VERSION="${PROMETHEUS_VERSION:-latest}" +PROMETHEUS_IMAGE="${DOCKERHUB_MIRROR}/prom/prometheus:${PROMETHEUS_VERSION}" +GRAFANA_VERSION="${GRAFANA_VERSION:-latest}" +GRAFANA_IMAGE="${DOCKERHUB_MIRROR}/grafana/grafana:${GRAFANA_VERSION}" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +if ! command -v kubectl &> /dev/null; then + log_error "kubectl not found. Please install kubectl first." + exit 1 +fi + +check_kpt() { + if ! command -v kpt &> /dev/null; then + log_error "kpt not found. Please install kpt from: https://kpt.dev/installation/" + exit 1 + fi +} + +prepare_manifests() { + local temp_dir=$(mktemp -d) + cp -r "${METRICS_DIR}"/* "$temp_dir/" + + cat > "$temp_dir/Kptfile" < /dev/null 2>&1 + + echo "$temp_dir" +} + +apply_manifests() { + local manifests_dir=$1 + + log_info "Applying manifests using kpt live apply..." + if [ ! -f "$manifests_dir/resourcegroup.yaml" ]; then + log_info "Initializing kpt inventory..." + kpt live init "$manifests_dir" --namespace "$NAMESPACE" --name porch-monitoring + fi + + log_info "Running kpt live apply..." + kpt live apply "$manifests_dir" --reconcile-timeout=2m --output=events || { + log_warn "kpt live apply reconcile timeout - resources are deployed but may still be starting up" + } +} + +create_namespace() { + if kubectl get namespace "$NAMESPACE" &> /dev/null; then + log_info "Namespace $NAMESPACE already exists" + else + log_info "Creating namespace $NAMESPACE" + kubectl create namespace "$NAMESPACE" + fi +} + +deploy_monitoring() { + log_info "Deploying monitoring stack..." + log_info "Rendering manifests with kpt..." + + local manifests_dir + manifests_dir=$(prepare_manifests) + + kubectl create configmap prometheus-config \ + --from-file="${SCRIPT_DIR}/../deployments/metrics-resources/prometheus-config.yaml" \ + -n "$NAMESPACE" \ + --dry-run=client -o yaml | kubectl apply -f - + + + declare -a grafana_dashboards + while read -r dashboard_file; do + grafana_dashboards+=("--from-file=$(basename "$dashboard_file")=$dashboard_file") + done < <(find "${SCRIPT_DIR}/../deployments/metrics-resources" -name "grafana*dashboard.json" -type f) + + kubectl create configmap grafana-dashboards \ + "${grafana_dashboards[@]}" \ + -n "$NAMESPACE" \ + --dry-run=client -o yaml | kubectl apply -f - + + apply_manifests "$manifests_dir" + + rm -rf "$manifests_dir" + + log_info "Monitoring stack deployed successfully" +} + +wait_for_deployment() { + local deployment=$1 + log_info "Waiting for $deployment to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/"$deployment" -n "$NAMESPACE" +} + +stop_port_forwards() { + if find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' -exec pkill -F '{}' \; 2>/dev/null; then + find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' ! -wholename "${PORT_FORWARD_DIR}*" -exec rm '{}' \; 2>/dev/null || true + find /tmp/tmp*_porch-monitoring-pf.pid.d/ -type d ! -wholename "${PORT_FORWARD_DIR}*" -exec rmdir '{}' \; 2>/dev/null || true + fi +} + +get_service_urls() { + log_info "Getting service URLs..." + + log_info "Setting up port forwarding..." + stop_port_forwards + sleep 2 + + kubectl port-forward -n "${NAMESPACE}" deployment/prometheus "${PROMETHEUS_LOCAL_PORT}":"${PROMETHEUS_CONTAINER_PORT}" > /dev/null 2>&1 & + PROMETHEUS_PF_PID=$! + kubectl port-forward -n "${NAMESPACE}" deployment/grafana "${GRAFANA_LOCAL_PORT}":"${GRAFANA_CONTAINER_PORT}" > /dev/null 2>&1 & + GRAFANA_PF_PID=$! + + sleep 2 + + echo "${PROMETHEUS_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-prometheus-pf.pid + echo "${GRAFANA_PF_PID}" > "${PORT_FORWARD_DIR}"/porch-grafana-pf.pid + + PROMETHEUS_URL="http://localhost:${PROMETHEUS_LOCAL_PORT}" + GRAFANA_URL="http://localhost:${GRAFANA_LOCAL_PORT}" + + echo "" + log_info "==========================================" + log_info "Services deployed successfully!" + log_info "==========================================" + echo "" + log_info "Access via port-forward (recommended):" + log_info " Prometheus: ${PROMETHEUS_URL}" + log_info " Grafana: ${GRAFANA_URL}" + log_info " Username: ${GRAFANA_ADMIN_USER}" + log_info " Password: ${GRAFANA_ADMIN_PW}" + log_info " stored in: kubectl -n porch-monitoring get secrets --selector app=grafana -o yaml" + echo "" + log_info " - Prometheus is scraping metrics from:" + log_info " - porch-server port 9464" + log_info " - porch-controller port 9464" + log_info " - function-runner port 9464" + log_info "" + echo "" + log_info "To stop port forwarding, run:" + log_info " find /tmp/tmp*_porch-monitoring-pf.pid.d/ -name '*.pid' -exec pkill -F '{}' \;" + echo "" +} + +cleanup() { + log_warn "Cleaning up existing deployment..." + + log_info "Stopping port forwarding..." + stop_port_forwards + + if kubectl get namespace "$NAMESPACE" &> /dev/null; then + log_info "Deleting resources in namespace $NAMESPACE..." + kubectl delete deployment prometheus grafana -n "$NAMESPACE" --ignore-not-found=true + kubectl delete service prometheus grafana -n "$NAMESPACE" --ignore-not-found=true + kubectl delete configmap prometheus-config grafana-dashboards grafana-dashboards-provider grafana-datasources -n "$NAMESPACE" --ignore-not-found=true + kubectl delete secret grafana-admin-creds -n "$NAMESPACE" --ignore-not-found=true + kubectl delete serviceaccount prometheus -n "$NAMESPACE" --ignore-not-found=true + kubectl delete clusterrole prometheus --ignore-not-found=true + kubectl delete clusterrolebinding prometheus --ignore-not-found=true + + log_info "Deleting namespace $NAMESPACE..." + kubectl delete namespace "$NAMESPACE" --ignore-not-found=true + else + log_info "Namespace $NAMESPACE does not exist, nothing to clean up" + fi + + log_info "Cleanup completed" +} + +main() { + local action="${1:-deploy}" + case "$action" in + deploy) + log_info "Starting deployment of Prometheus and Grafana..." + check_kpt + create_namespace + deploy_monitoring + wait_for_deployment prometheus + wait_for_deployment grafana + get_service_urls + ;; + cleanup) + check_kpt + cleanup + ;; + restart) + check_kpt + cleanup + sleep 2 + main deploy + ;; + *) + log_error "Unknown action: $action" + echo "Usage: $0 {deploy|cleanup|restart}" + echo "" + echo "Environment variables:" + echo " NAMESPACE - Kubernetes namespace (default: porch-monitoring)" + echo " PROMETHEUS_NODEPORT - Prometheus NodePort (default: 30091)" + echo " GRAFANA_NODEPORT - Grafana NodePort (default: 30301)" + echo "" + echo "Requirements:" + echo " - kpt CLI (install from: https://kpt.dev/installation/)" + echo " - kubectl configured with cluster access" + exit 1 + ;; + esac +} +main "$@" diff --git a/test/e2e/api/metrics_test.go b/test/e2e/api/metrics_test.go index 18cdc0858..f94201912 100644 --- a/test/e2e/api/metrics_test.go +++ b/test/e2e/api/metrics_test.go @@ -1,35 +1,39 @@ package api import ( + "slices" + + porchapi "github.com/kptdev/porch/api/porch/v1alpha1" suiteutils "github.com/kptdev/porch/test/e2e/suiteutils" - "github.com/stretchr/testify/assert" + "github.com/prometheus/common/model" + "sigs.k8s.io/controller-runtime/pkg/client" ) func (t *PorchSuite) TestMetricsEndpoint() { porchServerShouldHaveRegexList := []string{ - "go_*", + "go_.*", "http_server_.*", "http_client_.*", - "errors_total .*", - "target_info .*", + "errors_total.*", + "target_info.*", "promhttp_metric_handler_.*", } porchControllerShouldHaveRegexList := []string{ - "controller_*", - "go_*", + "controller_.*", + "go_.*", } porchFunctionRunnerShouldHaveRegexList := []string{ - "go_*", - "rpc_server_*", - // "rpc_client_*", //There is no way to force both function runners to have at least one connection, so no metrics + "go_.*", + "rpc_server_.*", + // "rpc_client_.*", //There is no way to force both function runners to have at least one connection, so no metrics } porchWrapperServerShouldHaveRegexList := []string{ - "go_*", - "rpc_server_*", + "go_.*", + "rpc_server_.*", } - //This is needed to ensure that there is at least one wrapper-server instance. - + // Create a package revision and update it with a mutator. + // This is needed to trigger a render and ensure that there is at least one wrapper-server instance. resources := t.setupFunctionTestPackage("git-fn-distroless", "test-fn-redis-bucket", "test-description", TestPackageSetupOptions{ UpstreamRef: "redis-bucket/v1", UpstreamDir: "redis-bucket", @@ -53,17 +57,133 @@ data: } for _, regex := range porchServerShouldHaveRegexList { - t.Regexp(regex, collectionResults.PorchServerMetrics, "porch server metrics should contain %q", regex) + t.Assert().Regexp(regex, collectionResults.PorchServerMetrics, "porch server metrics should contain %q", regex) } for _, regex := range porchControllerShouldHaveRegexList { - assert.Regexp(t.T(), regex, collectionResults.PorchControllerMetrics, "porch controller metrics should contain %q", regex) + t.Assert().Regexp(regex, collectionResults.PorchControllerMetrics, "porch controller metrics should contain %q", regex) } for _, regex := range porchFunctionRunnerShouldHaveRegexList { - assert.Regexp(t.T(), regex, collectionResults.PorchFunctionRunnerMetrics, "porch function runner metrics should contain %q", regex) + t.Assert().Regexp(regex, collectionResults.PorchFunctionRunnerMetrics, "porch function runner metrics should contain %q", regex) } for _, regex := range porchWrapperServerShouldHaveRegexList { - assert.Regexp(t.T(), regex, collectionResults.PorchWrapperServerMetrics, "porch wrapper server metrics should contain %q", regex) + t.Assert().Regexp(regex, collectionResults.PorchWrapperServerMetrics, "porch wrapper server metrics should contain %q", regex) + } +} + +const dbCacheSkipMessage = "Package size metrics are only supported in DB cache deployments. If you already deployed Porch with the DB cache activated, set the DB_CACHE environment variable and re-run this test." + +func (t *PorchSuite) TestPackageSizeMetric() { + if !t.UsingDBCache { + t.T().Skip(dbCacheSkipMessage) + } + + expectedMetrics := []string{ + `porch_package_size_bytes_bucket`, + `porch_package_size_bytes_count`, + `porch_package_size_bytes_sum`, + `porch_package_size_bytes_total`, + } + + // Create a new package revision to ensure metric creation in porch-server + t.setupFunctionTestPackage("git-fn-distroless", "test-fn-redis-bucket", "test-description", TestPackageSetupOptions{ + UpstreamRef: "redis-bucket/v1", + UpstreamDir: "redis-bucket", + }) + + collectionResults, err := t.CollectMetricsFromPods() + t.Require().NoError(err, "failed to collect metrics from pods:") + + for _, metricName := range expectedMetrics { + t.Assert().Regexp(metricName, collectionResults.PorchServerMetrics, "porch server metrics should contain %q", metricName) } } + +func (t *PorchSuite) TestPackageSizeMetricValues() { + if !t.UsingDBCache { + t.T().Skip(dbCacheSkipMessage) + } + + // Create a new package via init, no task specified + const ( + repository = "metrics-values" + packageName = "metrics-package" + workspace = "metrics-workspace" + description = "empty-package description" + + expectedMetric = "porch_package_size_bytes_total" + ) + + // initialize a package + resources := t.setupFunctionTestPackage(repository, packageName, workspace, TestPackageSetupOptions{ + UpstreamRef: "redis-bucket/v1", + UpstreamDir: "redis-bucket", + }) + resources.Spec.Resources["configmap.yaml"] = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: kptfile.kpt.dev +data: + name: bucket-namespace +` + + // push a resource change + t.AddMutator(resources, t.KrmFunctionsRegistry+"/"+setNamespaceImage, suiteutils.WithConfigPath("configmap.yaml")) + t.UpdateF(resources) + + pr := &porchapi.PackageRevision{} + t.GetF(client.ObjectKey{Namespace: t.Namespace, Name: resources.Name}, pr) + + t.validatePorchServerSizeMetric(pr, expectedMetric) + + // propose and approve + pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleProposed + t.UpdateF(pr) + pr.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished + pr = t.UpdateApprovalF(pr) + + t.validatePorchServerSizeMetric(pr, expectedMetric) + + // propose-delete and delete + pr.Spec.Lifecycle = porchapi.PackageRevisionLifecycleDeletionProposed + t.UpdateApprovalF(pr) + t.DeleteE(pr) + pr.Status.ResourcesSizeBytes = 0 + t.validatePorchServerSizeMetric(pr, expectedMetric) +} + +func (t *PorchSuite) validatePorchServerSizeMetric(pr *porchapi.PackageRevision, metricName string) { + t.T().Helper() + t.validateSizeMetric(pr, metricName, func(parsedResults *suiteutils.ParsedMetricsResults) map[string][]suiteutils.MetricResult { + return parsedResults.PorchServerMetrics + }) +} + +func (t *PorchSuite) validateSizeMetric(pr *porchapi.PackageRevision, metricName string, selectPodMetrics func(*suiteutils.ParsedMetricsResults) map[string][]suiteutils.MetricResult) { + t.T().Helper() + if t.UsingDBCache { + collectionResults, err := t.CollectMetricsFromPods() + t.Require().NoError(err, "failed to collect metrics from pods:") + parsedResults, err := collectionResults.Parse() + t.Require().NoError(err, "failed to parse collected metrics:") + + podParsedResults := selectPodMetrics(parsedResults) + + t.Assert().Contains(podParsedResults, metricName) + + metric := podParsedResults[metricName] + metric = slices.DeleteFunc(metric, func(aMetric suiteutils.MetricResult) bool { + return !(aMetric.Attributes["namespace"] == model.LabelValue(t.Namespace) && + aMetric.Attributes["repository"] == model.LabelValue(pr.Spec.RepositoryName) && + aMetric.Attributes["package"] == model.LabelValue(pr.Spec.PackageName) && + aMetric.Attributes["workspace_name"] == model.LabelValue(pr.Spec.WorkspaceName)) + }) + t.Require().Lenf(metric, 1, "Expected metrics to include exactly 1 %q entry with {namespace=%q, repository=%q, package=%q, workspace_name=%q}, but did not", metricName, t.Namespace, pr.Spec.RepositoryName, pr.Spec.PackageName, pr.Spec.WorkspaceName) + t.Assert().EqualValues(model.SampleValue(pr.Status.ResourcesSizeBytes), metric[0].Value) + } else { + t.Assert().EqualValues(0, pr.Status.ResourcesSizeBytes, "PackageRevision resources size should not be available in non-DB cache deployment") + } + +} diff --git a/test/e2e/suiteutils/suite.go b/test/e2e/suiteutils/suite.go index 2d5cc965e..5029b4ab0 100644 --- a/test/e2e/suiteutils/suite.go +++ b/test/e2e/suiteutils/suite.go @@ -160,10 +160,7 @@ func (t *TestSuite) Initialize() { } func (t *TestSuite) checkIfUsingDBCache() { - t.UsingDBCache = func() bool { - _, envVarSet := os.LookupEnv("DB_CACHE") - return envVarSet - }() + _, t.UsingDBCache = os.LookupEnv("DB_CACHE") } func (t *TestSuite) PorchServerServiceKey() client.ObjectKey { @@ -360,7 +357,7 @@ func (t *TestSuite) delete(obj client.Object, opts []client.DeleteOption, eh Err t.T().Helper() t.Logf("deleting object %v", DebugFormat(obj)) - if err := t.Client.Delete(t.GetContext(), obj, opts...); err != nil { + if err := client.IgnoreNotFound(t.Client.Delete(t.GetContext(), obj, opts...)); err != nil { eh("failed to delete resource %s: %v", DebugFormat(obj), err) } } diff --git a/test/e2e/suiteutils/suite_utils.go b/test/e2e/suiteutils/suite_utils.go index 7d23d524d..eacd304bc 100644 --- a/test/e2e/suiteutils/suite_utils.go +++ b/test/e2e/suiteutils/suite_utils.go @@ -34,6 +34,8 @@ import ( configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" pvapi "github.com/kptdev/porch/controllers/packagevariants/api/v1alpha1" internalapi "github.com/kptdev/porch/internal/api/porchinternal/v1alpha1" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" coreapi "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -64,6 +66,63 @@ type MetricsCollectionResults struct { PorchWrapperServerMetrics string } +type ParsedMetricsResults struct { + PorchServerMetrics map[string][]MetricResult + PorchControllerMetrics map[string][]MetricResult + PorchFunctionRunnerMetrics map[string][]MetricResult + PorchWrapperServerMetrics map[string][]MetricResult +} + +type MetricResult struct { + Value model.SampleValue + Attributes model.LabelSet +} + +func (r *MetricsCollectionResults) Parse() (*ParsedMetricsResults, error) { + parsed := &ParsedMetricsResults{ + PorchServerMetrics: make(map[string][]MetricResult), + PorchControllerMetrics: make(map[string][]MetricResult), + PorchFunctionRunnerMetrics: make(map[string][]MetricResult), + PorchWrapperServerMetrics: make(map[string][]MetricResult), + } + metricsParser := expfmt.NewTextParser(model.LegacyValidation) + + for _, pair := range []struct { + raw string + parsedResult map[string][]MetricResult + nameForError string + }{ + {r.PorchServerMetrics, parsed.PorchServerMetrics, "PorchServerMetrics"}, + {r.PorchControllerMetrics, parsed.PorchControllerMetrics, "PorchControllerMetrics"}, + {r.PorchFunctionRunnerMetrics, parsed.PorchFunctionRunnerMetrics, "PorchFunctionRunnerMetrics"}, + {r.PorchWrapperServerMetrics, parsed.PorchWrapperServerMetrics, "PorchWrapperServerMetrics"}, + } { + if pair.raw == "" { + continue + } + + metricFamilies, err := metricsParser.TextToMetricFamilies(strings.NewReader(pair.raw)) + if err != nil { + return nil, fmt.Errorf("error extracting metrics from %s text: %w", pair.nameForError, err) + } + + for metricName, family := range metricFamilies { + samples, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{}, family) + if err != nil { + return nil, fmt.Errorf("error extracting %s metric sample for %q: %w", pair.nameForError, metricName, err) + } + for _, sample := range samples { + pair.parsedResult[metricName] = append(pair.parsedResult[metricName], MetricResult{ + Value: sample.Value, + Attributes: model.LabelSet(sample.Metric), + }) + } + } + } + + return parsed, nil +} + type TestSuiteWithGit struct { TestSuite gitConfig GitConfig