Grafana dashboard generator tool

Initial commit for the `generate-rust-dashboards` tool.  This
auto-generates Grafana dashboards for teams that own Rust Components.
There's still a lot of work to do, but this feels like a good start.

Author: bendk

.cargo/config.toml

@@ -1,4 +1,5 @@
 [alias]
+generate-rust-dashboards = ["run", "-p", "generate-rust-dashboards"]
 regen-protobufs = ["run", "--bin", "protobuf-gen", "tools/protobuf_files.toml"]
 uniffi-bindgen = ["run", "--package", "embedded-uniffi-bindgen", "--"]
 uniffi-bindgen-library-mode = ["run", "--package", "uniffi-bindgen-library-mode", "--"]

Cargo.lock

@@ -61,6 +61,7 @@ members = [
     "megazords/ios-rust",
     "megazords/ios-rust/focus",
     "tools/embedded-uniffi-bindgen",
+    "tools/generate-rust-dashboards",
     "tools/start-bindings",
     "tools/uniffi-bindgen-library-mode",
     "automation/swift-components-docs",

Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "generate-rust-dashboards"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1"
+camino = "1"
+clap = {version = "4.2", default-features = false, features = ["std", "derive"]}
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"

tools/generate-rust-dashboards/Cargo.toml

@@ -0,0 +1,20 @@
+# Dashboard Generator
+
+Use this tool to create Grafana dashboards for your team's components.
+
+## Setup
+
+Ensure you have a yardstick account by going to https://yardstick.mozilla.org/ and logging in using Mozilla SSO.
+You should have “editor” access and can create, edit, and delete dashboards and alerts. If not, go
+to https://mozilla-hub.atlassian.net/wiki/spaces/SRE/pages/886866077/Yardstick+Grafana+Service+User+Guide
+for help.
+
+## Configuration
+
+Edit `src/component_config.rs` add a `Component` variant for each of your team's components.
+Edit `src/team_config.rs` and add an entry for your team.
+Feel free to copy the and paste other team's configurations to get started.
+
+## Running
+
+Run `cargo generate-rust-dashboards [team-name] [output-directory]` and follow the instructions.

tools/generate-rust-dashboards/README.md

@@ -0,0 +1,79 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use crate::config::{Application, Application::*};
+
+/// Enumeration containing all Rust components.
+/// When adding new variants, make sure to also update the impl block below
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Component {
+    Autofill,
+    Fxa,
+    Logins,
+    Places,
+    RemoteSettings,
+    Suggest,
+    Tabs,
+}
+
+impl Component {
+    /// Unique name for the component in slug format (lower-case letters + dashes).
+    pub fn slug(&self) -> &'static str {
+        match self {
+            Self::Autofill => "autofill",
+            Self::Fxa => "fxa",
+            Self::Logins => "logins",
+            Self::Places => "places",
+            Self::RemoteSettings => "remote-settings",
+            Self::Suggest => "suggest",
+            Self::Tabs => "tabs",
+        }
+    }
+
+    /// Applications your component ships on
+    pub fn applications(&self) -> &[Application] {
+        match self {
+            Self::Autofill => &[Android, Ios],
+            Self::Fxa => &[Android, Ios],
+            Self::Logins => &[Desktop, Android, Ios],
+            Self::Places => &[Android, Ios],
+            Self::RemoteSettings => &[Desktop, Android, Ios],
+            Self::Suggest => &[Desktop, Android, Ios],
+            Self::Tabs => &[Desktop, Android, Ios],
+        }
+    }
+
+    /// Prefix for error strings.
+    ///
+    /// This is the common prefix for strings sent to the `error_support`.  You can usually find it
+    /// by going to `error.rs` for your component and looking at the `report_error` calls.
+    pub fn error_prefix(&self) -> &'static str {
+        match self {
+            Self::Autofill => "autofill-",
+            Self::Fxa => "fxa-client-",
+            Self::Logins => "logins-",
+            Self::Places => "places-",
+            Self::RemoteSettings => "remote-settings-",
+            Self::Suggest => "suggest-",
+            Self::Tabs => "tabs-",
+        }
+    }
+
+    /// Sync engine names
+    ///
+    /// These represent 2 things:
+    ///   - The Glean pings for the component without the `-sync` suffix.
+    ///   - The `engine.name` value for the legacy `telemetry.sync` table.
+    pub fn sync_engines(&self) -> &[&'static str] {
+        match self {
+            Self::Autofill => &["addresses", "creditcards"],
+            Self::Fxa => &[],
+            Self::Logins => &["logins"],
+            Self::Places => &["bookmarks", "history"],
+            Self::RemoteSettings => &[],
+            Self::Suggest => &[],
+            Self::Tabs => &["tabs"],
+        }
+    }
+}

tools/generate-rust-dashboards/src/component_config.rs

@@ -0,0 +1,250 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use std::{collections::BTreeSet, fmt};
+
+pub use crate::component_config::Component;
+use crate::util::slug;
+
+/// Dashboard configuration for a team
+pub struct TeamConfig {
+    /// Display name for the team.
+    ///
+    /// This is what shows up on your dashboard titles.  Spell it out however you want.
+    pub team_name: &'static str,
+
+    /// Components that your team manages.
+    pub components: Vec<Component>,
+
+    /// Track component errors
+    ///
+    /// This adds a panel to the main dashboard as well as creates a extra dashboard for error
+    /// details.
+    pub component_errors: bool,
+
+    /// Track sync metrics
+    ///
+    /// This adds a panel to the main dashboard as well as creates a extra dashboard for sync
+    /// errors.
+    pub sync_metrics: bool,
+
+    /// Metric to include on your main dashboard
+    pub main_dashboard_metrics: Vec<Metric>,
+
+    /// Extra dashboards to generate
+    pub extra_dashboards: Vec<ExtraDashboard>,
+}
+
+/// Extra dashboard to generate for a team
+pub struct ExtraDashboard {
+    pub name: &'static str,
+    /// Metrics to include in the dashboard
+    pub metrics: Vec<Metric>,
+}
+
+/// Metric to add to your team dashboard
+///
+/// Metrics will add panels to your overview dashboard and/or create secondary dashboards.
+pub enum Metric {
+    Counter(CounterMetric),
+    LabeledCounter(LabeledCounterMetric),
+    Distribution(DistributionMetric),
+    LabeledDistribution(LabeledDistributionMetric),
+}
+
+/// Glean counter
+///
+/// This will create time-series panels for the counter
+pub struct CounterMetric {
+    /// Name to display on the dashboard
+    pub display_name: &'static str,
+    /// Name of the ping ("metrics" by default)
+    pub ping: &'static str,
+    /// Category name (top-level key in metrics.yaml)
+    pub category: &'static str,
+    /// Metric name (key for the metric)
+    pub metric: &'static str,
+    // Which applications report this metric
+    pub applications: Vec<Application>,
+}
+
+/// Glean labeled counter
+///
+/// This will create time-series panels for the counter, partitioned by the label
+pub struct LabeledCounterMetric {
+    /// Name to display on the dashboard
+    pub display_name: &'static str,
+    /// Name of the ping ("metrics" by default)
+    pub ping: &'static str,
+    /// Category name (top-level key in metrics.yaml)
+    pub category: &'static str,
+    /// Metric name (key for the metric)
+    pub metric: &'static str,
+    // Which applications report this metric
+    pub applications: Vec<Application>,
+}
+
+/// Glean timing/memory distribution
+///
+/// This will create time-series panels for the 5th, 50th and 95th percentile.
+pub struct DistributionMetric {
+    pub kind: DistributionMetricKind,
+    /// Name to display on the dashboard
+    pub display_name: &'static str,
+    /// Label describing what we're measure, including units
+    pub axis_label: &'static str,
+    /// Name of the ping ("metrics" by default)
+    pub ping: &'static str,
+    /// Category name (top-level key in metrics.yaml)
+    pub category: &'static str,
+    /// Metric name (key for the metric)
+    pub metric: &'static str,
+    // Which applications report this metric
+    pub applications: Vec<Application>,
+    // Divide each value by this amount
+    //
+    // Note:
+    // * Timing distributions are always stored in nanoseconds, regardless of the unit listed in
+    //   `metrics.yaml`
+    // * Memory distributions are always stored in bytes, regardless of the unit listed in
+    //   `metrics.yaml`
+    pub value_divisor: Option<u64>,
+    // Filter out values lower than this amount (takes effect before the divisor)
+    pub value_filter: Option<u64>,
+    // Link to an extra dashboard, the inner value is the name of the dashboard
+    pub link_to: Option<&'static str>,
+}
+
+/// Glean labeled timing/memory distribution
+///
+/// This will create time-series panels for the 5th, 50th and 95th percentile.
+/// Percentiles will be partitioned by the metric label.
+pub struct LabeledDistributionMetric {
+    pub kind: DistributionMetricKind,
+    /// Name to display on the dashboard
+    pub display_name: &'static str,
+    /// Label describing what we're measure, including units
+    pub axis_label: &'static str,
+    /// Name of the ping ("metrics" by default)
+    pub ping: &'static str,
+    /// Category name (top-level key in metrics.yaml)
+    pub category: &'static str,
+    /// Metric name (key for the metric)
+    pub metric: &'static str,
+    // Which applications report this metric
+    pub applications: Vec<Application>,
+    // Divide each value by this amount
+    //
+    // Note:
+    // * Timing distributions are always stored in nanoseconds, regardless of the unit listed in
+    //   `metrics.yaml`
+    // * Memory distributions are always stored in bytes, regardless of the unit listed in
+    //   `metrics.yaml`
+    pub value_divisor: Option<u64>,
+    // Filter out values lower than this amount (takes effect before the divisor)
+    pub value_filter: Option<u64>,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum DistributionMetricKind {
+    Memory,
+    Timing,
+    Custom,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Application {
+    Android,
+    Ios,
+    Desktop,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum ReleaseChannel {
+    Nightly,
+    Beta,
+    Release,
+}
+
+impl TeamConfig {
+    pub fn applications(&self) -> BTreeSet<Application> {
+        self.components
+            .iter()
+            .flat_map(Component::applications)
+            .cloned()
+            .collect()
+    }
+
+    pub fn team_slug(&self) -> String {
+        slug(self.team_name)
+    }
+}
+
+impl Application {
+    pub fn slug(&self) -> &'static str {
+        match self {
+            Self::Android => "android",
+            Self::Ios => "ios",
+            Self::Desktop => "desktop",
+        }
+    }
+
+    pub fn bigquery_dataset(&self) -> &'static str {
+        // There's a few datasets we can use, these were chosen because they seem to include
+        // release, beta, and nightly data
+        match self {
+            Self::Android => "fenix",
+            Self::Ios => "firefox_ios",
+            Self::Desktop => "firefox_desktop",
+        }
+    }
+
+    pub fn display_name(&self, channel: ReleaseChannel) -> String {
+        format!("{self} ({channel})")
+    }
+}
+
+impl fmt::Display for Application {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Android => write!(f, "Android"),
+            Self::Ios => write!(f, "iOS"),
+            Self::Desktop => write!(f, "Desktop"),
+        }
+    }
+}
+
+impl fmt::Display for ReleaseChannel {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Nightly => write!(f, "nightly"),
+            Self::Beta => write!(f, "beta"),
+            Self::Release => write!(f, "release"),
+        }
+    }
+}
+
+impl From<CounterMetric> for Metric {
+    fn from(m: CounterMetric) -> Self {
+        Self::Counter(m)
+    }
+}
+
+impl From<LabeledCounterMetric> for Metric {
+    fn from(m: LabeledCounterMetric) -> Self {
+        Self::LabeledCounter(m)
+    }
+}
+
+impl From<DistributionMetric> for Metric {
+    fn from(m: DistributionMetric) -> Self {
+        Self::Distribution(m)
+    }
+}
+
+impl From<LabeledDistributionMetric> for Metric {
+    fn from(m: LabeledDistributionMetric) -> Self {
+        Self::LabeledDistribution(m)
+    }
+}