roachprod,cli: metric name consolidation in tsdump upload

Previously, tsdump was uploading time series data with metrics name defined in
registry. This was inadequate because same metric has different name in the
Datadog integration based on offering. We are working on consolidating
observability stack across our offerings. To address this, this patch updates
the tsdump upload functionality generate metrics renaming during execution. We
have updated gen-cockroachdb-metrics tool to generate yaml files. The
generation will support comprehensive and datadog only mode. In the
comprehensive mode, we are generating crdb metrics renaming mapping. The
datadog only mode would only generate the datadog integration metric renaming
mapping file.  We had to created datadog only mode to avoid circular dependency
on package cli.

Part of: CRDB-56324
Epic: None
Release note: None

Author: aa-joshi

build/tools/gen-cockroachdb-metrics/main.go

@@ -3,7 +3,7 @@
 // Use of this software is governed by the CockroachDB Software License
 // included in the /LICENSE file.
 
-// gen-cockroachdb-metrics generates the cockroachdb_metrics.go file for roachprod's OpenTelemetry
+// gen-cockroachdb-metrics generates the cockroachdb_metrics.yaml file for roachprod's OpenTelemetry
 // integration, enabling proper metric forwarding to Datadog from CockroachDB clusters.
 //
 // This generator creates a comprehensive mapping of CockroachDB metric names to Datadog-compatible
@@ -31,7 +31,7 @@
 //	c) Processes metrics.yaml and runtime_conditional_metrics, converting them to Prometheus format and applying
 //	   Datadog naming conventions (including histogram variants like _bucket, _count, _sum)
 //	d) Merges the processed mappings with legacy_metrics
-//	e) Generates cockroachdb_metrics.go containing the final consolidated mapping
+//	e) Generates cockroachdb_metrics.yaml containing the final consolidated mapping
 
 package main
 
@@ -108,43 +108,63 @@ var (
 	prometheusNameRegex = regexp.MustCompile(`[^a-zA-Z0-9]`)
 )
 
-// Go code template for generating the cockroachdb_metrics.go file
-const goTemplate = `// Copyright 2024 The Cockroach Authors.
-//
-// Use of this software is governed by the CockroachDB Software License
-// included in the /LICENSE file.
-//
-// DO NOT EDIT THIS FILE MANUALLY
-//
-// This file is auto-generated when running "./dev generate".
-// Manual edits will be overwritten.
-//
-// Generator: build/tools/gen-cockroachdb-metrics
-//
-// This file is generated by merging metrics from three sources:
-//   1. runtime_conditional_metrics from cockroachdb_metrics_base.yaml
-//      - Conditionally emitted metrics not documented in metrics.yaml
-//      - Processed through Datadog mapping rules
-//   2. legacy_metrics from cockroachdb_metrics_base.yaml
-//      - Legacy metrics from older CRDB versions for backward compatibility
-//      - Used as-is
-//   3. docs/generated/metrics/metrics.yaml
-//      - All current metrics documented in the latest CRDB version
-//      - Processed through Datadog mapping rules
-
-package opentelemetry
-
-// cockroachdbMetrics is a mapping of CockroachDB metric names to cockroachdb
-// Datadog integration metric names. This allows CockroachDB metrics to comply
-// with the naming requirements for the cockroachdb Datadog integration listed
-// in its metadata.csv.
-// - https://github.com/DataDog/integrations-core/blob/master/cockroachdb/metadata.csv
-
-var cockroachdbMetrics = map[string]string{
+// YAML template for generating the cockroachdb_metrics.yaml file
+const yamlTemplate = `# Copyright 2024 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+#
+# DO NOT EDIT THIS FILE MANUALLY
+#
+# This file is auto-generated when running "./dev generate".
+# Manual edits will be overwritten.
+#
+# Generator: build/tools/gen-cockroachdb-metrics
+#
+# This file is generated by merging metrics from three sources:
+#   1. runtime_conditional_metrics from cockroachdb_metrics_base.yaml
+#      - Conditionally emitted metrics not documented in metrics.yaml
+#      - Processed through Datadog mapping rules
+#   2. legacy_metrics from cockroachdb_metrics_base.yaml
+#      - Legacy metrics from older CRDB versions for backward compatibility
+#      - Used as-is
+#   3. docs/generated/metrics/metrics.yaml
+#      - All current metrics documented in the latest CRDB version
+#      - Processed through Datadog mapping rules
+#
+# cockroachdbMetrics is a mapping of CockroachDB metric names to cockroachdb
+# Datadog integration metric names. This allows CockroachDB metrics to comply
+# with the naming requirements for the cockroachdb Datadog integration listed
+# in its metadata.csv.
+# - https://github.com/DataDog/integrations-core/blob/master/cockroachdb/metadata.csv
+
 {{- range $key, $value := .Metrics }}
-    "{{ $key }}": "{{ $value }}",
+{{ $key }}: {{ $value }}
+{{- end }}
+`
+
+// YAML template for generating the raw Datadog mappings YAML file
+const datadogMappingsTemplate = `# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+#
+# DO NOT EDIT THIS FILE MANUALLY
+#
+# This file is auto-generated when running "./dev generate".
+# Manual edits will be overwritten.
+#
+# Generator: build/tools/gen-cockroachdb-metrics
+#
+# Source: https://github.com/DataDog/integrations-core/blob/master/cockroachdb/datadog_checks/cockroachdb/metrics.py
+#
+# This file contains the raw Datadog metric name mappings from METRIC_MAP and OMV2_METRIC_MAP.
+# The key is the CockroachDB metric name in Prometheus format, and the value is the
+# corresponding metric name as it appears in Datadog.
+
+{{- range $key, $value := .Metrics }}
+{{ $key }}: {{ $value }}
 {{- end }}
-}
 `
 
 // loadDatadogMappings loads the Datadog metrics mapping from a local file
@@ -170,9 +190,9 @@ func parseDatadogMappings(r io.Reader) (map[string]string, error) {
 
 	// CRDB-Datadog mappings are stored as python dictionaries in the following file:
 	// https://github.com/DataDog/integrations-core/blob/master/cockroachdb/datadog_checks/cockroachdb/metrics.py
-	//   - METRIC_MAP: represents the raw CRDB-Datadog metric name mapping. 
+	//   - METRIC_MAP: represents the raw CRDB-Datadog metric name mapping.
 	//   - OMV2_METRIC_MAP: represents the metric in OpenMetrics V2 format.
-	// E.g. 
+	// E.g.
 	// 'admission_errored_sql_kv_response': 'admission.errored.sql_kv.response'
 	// here the key is the CRDB metric name in prometheus format, and the value is the corresponding metric name visible in Datadog.
 	// Both maps are mutually exclusive. Parse both dictionaries to get the complete mapping.
@@ -214,12 +234,17 @@ func parsePythonDict(lines []string, dictName string) (map[string]string, error)
 			break
 		}
 
+		// Skip commented lines
+		if strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+
 		// Parse metric mapping line: 'key': 'value',
 		if strings.Contains(trimmed, ":") && strings.Contains(trimmed, "'") {
 			parts := strings.SplitN(trimmed, ":", 2)
 			if len(parts) == 2 {
-				key := strings.Trim(strings.TrimSpace(parts[0]), "'\"")
-				value := strings.Trim(strings.TrimSpace(parts[1]), "',\"")
+				key := strings.Trim(strings.TrimSpace(parts[0]), " '\"")
+				value := strings.Trim(strings.TrimSpace(parts[1]), " ',\"")
 				if key != "" && value != "" {
 					metrics[strings.ToLower(key)] = value
 				}
@@ -348,9 +373,47 @@ func mergeLegacyMappings(legacyMappings, newMappings map[string]string) map[stri
 	return result
 }
 
-// writeGoFile generates the cockroachdb_metrics.go file
-func writeGoFile(outputPath string, metrics map[string]string) error {
-	tmpl, err := template.New("cockroachdb_metrics").Parse(goTemplate)
+// writeYAMLFile generates the cockroachdb_metrics.yaml file
+func writeYAMLFile(outputPath string, metrics map[string]string) error {
+	tmpl, err := template.New("cockroachdb_metrics").Parse(yamlTemplate)
+	if err != nil {
+		return fmt.Errorf("parsing template: %w", err)
+	}
+
+	// Sort keys for consistent output
+	keys := make([]string, 0, len(metrics))
+	for k := range metrics {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	sortedMetrics := make(map[string]string)
+	for _, k := range keys {
+		sortedMetrics[k] = metrics[k]
+	}
+
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return fmt.Errorf("creating file: %w", err)
+	}
+	defer file.Close()
+
+	data := struct {
+		Metrics map[string]string
+	}{
+		Metrics: sortedMetrics,
+	}
+
+	if err := tmpl.Execute(file, data); err != nil {
+		return fmt.Errorf("executing template: %w", err)
+	}
+
+	return nil
+}
+
+// writeDatadogMappingsYAML generates the raw Datadog mappings YAML file
+func writeDatadogMappingsYAML(outputPath string, metrics map[string]string) error {
+	tmpl, err := template.New("datadog_mappings").Parse(datadogMappingsTemplate)
 	if err != nil {
 		return fmt.Errorf("parsing template: %w", err)
 	}
@@ -386,9 +449,51 @@ func writeGoFile(outputPath string, metrics map[string]string) error {
 	return nil
 }
 
+func printUsage() {
+	fmt.Fprintf(os.Stderr, "Usage:\n")
+	fmt.Fprintf(os.Stderr, "  Generate CockroachDB metrics with Datadog mappings:\n")
+	fmt.Fprintf(os.Stderr, "    %s <metrics.yaml> <output.yaml> <datadog-mappings.py> [base.yaml]\n\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "  Generate raw Datadog mappings only:\n")
+	fmt.Fprintf(os.Stderr, "    %s --datadog-only <datadog-mappings.py> <output.yaml>\n", os.Args[0])
+}
+
 func main() {
+	if len(os.Args) < 2 {
+		printUsage()
+		os.Exit(1)
+	}
+
+	// Check if running in datadog-only mode
+	if os.Args[1] == "--datadog-only" {
+		if len(os.Args) < 4 {
+			printUsage()
+			os.Exit(1)
+		}
+
+		datadogMappingsPath := os.Args[2]
+		outputPath := os.Args[3]
+
+		datadogMappings, err := loadDatadogMappings(datadogMappingsPath)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "ERROR:", err)
+			os.Exit(1)
+		}
+
+		if err := writeDatadogMappingsYAML(outputPath, datadogMappings); err != nil {
+			fmt.Fprintln(os.Stderr, "ERROR:", err)
+			os.Exit(1)
+		}
+
+		if !strings.Contains(outputPath, "-exec-") && !strings.Contains(outputPath, "for tool") {
+			fmt.Printf("Generated %d raw Datadog metric mappings\n", len(datadogMappings))
+			fmt.Printf("Output written to: %s\n", outputPath)
+		}
+		return
+	}
+
+	// Normal mode: generate cockroachdb_metrics.yaml
 	if len(os.Args) < 4 {
-		fmt.Fprintf(os.Stderr, "Usage: %s <metrics.yaml> <output.go> <datadog-mappings.py> [base.yaml]\n", os.Args[0])
+		printUsage()
 		os.Exit(1)
 	}
 
@@ -424,7 +529,7 @@ func main() {
 	newMappings := mapMetricsToDatadog(allMetrics, datadogMappings)
 	finalMappings := mergeLegacyMappings(baseMetrics.LegacyMetrics, newMappings)
 
-	if err := writeGoFile(outputPath, finalMappings); err != nil {
+	if err := writeYAMLFile(outputPath, finalMappings); err != nil {
 		fmt.Fprintln(os.Stderr, "ERROR:", err)
 		os.Exit(1)
 	}

pkg/cli/BUILD.bazel

@@ -2,6 +2,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 load("//build:STRINGER.bzl", "stringer")
 load("//pkg/testutils:buildutil/buildutil.bzl", "disallowed_imports_test")
 
+genrule(
+    name = "cockroachdb_datadog_metrics",
+    srcs = ["@datadog_cockroachdb_metrics//:metrics.py"],
+    outs = ["files/cockroachdb_datadog_metrics.yaml"],
+    cmd = "$(location //build/tools/gen-cockroachdb-metrics) --datadog-only $(location @datadog_cockroachdb_metrics//:metrics.py) $@",
+    tools = ["//build/tools/gen-cockroachdb-metrics"],
+    visibility = ["//pkg/gen:__pkg__"],
+)
+
 go_library(
     name = "cli",
     srcs = [
@@ -97,6 +106,10 @@ go_library(
     cppopts = [
         "-DJEMALLOC_NO_DEMANGLE",
     ],
+    embedsrcs = [
+        "files/cockroachdb_metrics_base.yaml",
+        "files/cockroachdb_datadog_metrics.yaml",
+    ],
     importpath = "github.com/cockroachdb/cockroach/pkg/cli",
     visibility = ["//visibility:public"],
     deps = [
@@ -231,6 +244,7 @@ go_library(
         "//pkg/util/tracing/tracingpb",
         "//pkg/util/tracing/zipper",
         "//pkg/util/uuid",
+        "//pkg/util/yamlutil",
         "//pkg/workload",
         "//pkg/workload/bank",
         "//pkg/workload/bulkingest",