feat: own the dbt Semantic Layer client, talking to the API directly

AGENTS.md

@@ -26,6 +26,7 @@ lfx-mcp/
 │   └── lfx-mcp-server/     # Main application entry point
 ├── internal/
 │   ├── auth/               # JWT and API-key verification
+│   ├── dbtsl/              # dbt Semantic Layer client (GraphQL)
 │   ├── lfxv2/              # LFX V2 API client
 │   ├── otel/               # OpenTelemetry instrumentation
 │   ├── serviceapi/         # Shared service API helpers
@@ -490,6 +491,39 @@ The server supports configuration via environment variables with the `LFXMCP_` p
 | `-onboarding_api_audience`      | `LFXMCP_ONBOARDING_API_AUDIENCE`      | —              | Auth0 resource server audience for the member onboarding API      |
 | `-lens_api_url`                 | `LFXMCP_LENS_API_URL`                 | —              | Base URL of the LFX Lens service                                  |
 | `-lens_api_audience`            | `LFXMCP_LENS_API_AUDIENCE`            | —              | Auth0 resource server audience for the LFX Lens API               |
+| `-dbt_sl_host`                  | `LFXMCP_DBT_SL_HOST`                  | —              | dbt Semantic Layer host, without scheme                           |
+| `-dbt_sl_environment_id`        | `LFXMCP_DBT_SL_ENVIRONMENT_ID`        | —              | dbt Semantic Layer environment ID                                 |
+| `-dbt_sl_token`                 | `LFXMCP_DBT_SL_TOKEN`                 | —              | dbt Semantic Layer service token                                  |
+
+### Two data paths, two services
+
+`query_lfx_lens` and the semantic layer tools answer overlapping questions but
+do not share a backend, and the settings above reflect that:
+
+- **`query_lfx_lens`** posts a natural-language question to the LFX Lens
+  service, which generates SQL. It needs `LFXMCP_LENS_API_*`, and its client is
+  built through `internal/serviceapi` with an Auth0 client-credentials token.
+- **`explore_lfx_semantic_layer` and `query_lfx_semantic_layer`** talk to the
+  dbt Semantic Layer directly through `internal/dbtsl`. They need
+  `LFXMCP_DBT_SL_*` and nothing else: a static service token, no Auth0. The
+  client is therefore constructed in its own top-level block in `main.go`
+  rather than inside the LFX API block, so it does not become unconfigured for
+  an unrelated reason.
+
+`internal/dbtsl` uses the GraphQL API for both metadata and query execution.
+That diverges from the Python reference implementations (lfx-lens and dbt Labs'
+`dbt-mcp`), which run queries over Arrow Flight through the `dbtsl` SDK; there
+is no Go SDK, and Arrow plus gRPC buys nothing at a 500-row ceiling. A live
+parity harness sits behind the `parity` build tag:
+
+```bash
+set -a && source ../lfx-lens/.env && set +a
+go test -tags parity -v ./internal/dbtsl/
+```
+
+Do not pass the `serviceapi` debug transport to the dbt client. It dumps the
+`Authorization` header, and production runs with `debugTraffic` enabled, so it
+would print the long-lived service token into the logs.
 
 ## Error Handling Patterns
 

README.md

@@ -282,11 +282,20 @@ Hitting **Connect** will open a browser window for LFID login.
 
 ### LFX Lens
 
-| Tool                         | Description                                                                                           |
-|------------------------------|-------------------------------------------------------------------------------------------------------|
-| `query_lfx_lens`             | Ask natural-language questions about a project's data (events, contributors, health, value, and more) |
-| `explore_lfx_semantic_layer` | Discover Insights metrics and the dimensions available to them                                        |
-| `query_lfx_semantic_layer`   | Run a metric query against the Insights Semantic Layer (filter, group, rank, trend)                   |
+| Tool             | Description                                                                                           |
+|------------------|-------------------------------------------------------------------------------------------------------|
+| `query_lfx_lens` | Ask natural-language questions about a project's data (events, contributors, health, value, and more) |
+
+### Insights Semantic Layer
+
+Governed metrics with named dimensions, queried against the dbt Semantic Layer.
+Prefer these over `query_lfx_lens` for anything that reduces to a metric: the
+answer is repeatable and auditable, where generated SQL is neither.
+
+| Tool                         | Description                                                                         |
+|------------------------------|-------------------------------------------------------------------------------------|
+| `explore_lfx_semantic_layer` | Discover metrics, the dimensions available to them, and the values a dimension holds |
+| `query_lfx_semantic_layer`   | Run a metric query (filter, group, rank, trend)                                      |
 
 ### B2B Organizations
 

charts/lfx-mcp/templates/deployment.yaml

@@ -80,6 +80,14 @@ spec:
             - name: LFXMCP_LENS_API_AUDIENCE
               value: {{ .Values.app.lensApiAudience | quote }}
             {{- end }}
+            {{- if .Values.app.dbtSlHost }}
+            - name: LFXMCP_DBT_SL_HOST
+              value: {{ .Values.app.dbtSlHost | quote }}
+            {{- end }}
+            {{- if .Values.app.dbtSemanticEnvironmentId }}
+            - name: LFXMCP_DBT_SL_ENVIRONMENT_ID
+              value: {{ .Values.app.dbtSemanticEnvironmentId | quote }}
+            {{- end }}
             {{- if .Values.app.memberOnboardingApiUrl }}
             - name: LFXMCP_ONBOARDING_API_URL
               value: {{ .Values.app.memberOnboardingApiUrl | quote }}
@@ -105,6 +113,12 @@ spec:
                   name: {{ .Values.secrets.name }}
                   key: {{ .Values.secrets.keys.clientAssertionSigningKey }}
                   optional: true
+            - name: LFXMCP_DBT_SL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.secrets.name }}
+                  key: {{ .Values.secrets.keys.dbtSemanticServiceToken }}
+                  optional: true
             {{- /*
               TEMPORARY: Expose each API key credential as an individual env var
               LFXMCP_API_CREDENTIALS_<KEY>=<secret> sourced from the Secret referenced by

charts/lfx-mcp/values.yaml

@@ -100,6 +100,13 @@ app:
   lensApiUrl: ""
   # lensApiAudience is the Auth0 resource server audience for the LFX Lens API.
   lensApiAudience: ""
+  # dbtSlHost is the dbt Semantic Layer host, without scheme,
+  # e.g. tj283.semantic-layer.us1.dbt.com. Set it together with
+  # dbtSemanticEnvironmentId and the token; the semantic layer tools return an
+  # error when any of the three is missing.
+  dbtSlHost: ""
+  # dbtSemanticEnvironmentId is the dbt environment the semantic layer queries.
+  dbtSemanticEnvironmentId: ""
   # memberOnboardingApiUrl is the base URL of the member onboarding service.
   memberOnboardingApiUrl: ""
   # memberOnboardingApiAudience is the Auth0 resource server audience for the member onboarding API.
@@ -147,6 +154,10 @@ secrets:
     # assertions (used when authenticating with a JWT client assertion instead of a
     # client secret).
     clientAssertionSigningKey: client_private_key
+    # dbtSemanticServiceToken is the key name for the dbt Semantic Layer service
+    # token. It arrives from the AWS Secrets Manager secret tagged
+    # service-lfx-mcp, which lfx-lens also reads.
+    dbtSemanticServiceToken: dbt_semantic_service_token
 # apiCredentials is a TEMPORARY stop-gap for MCP clients that cannot complete a full
 # OAuth2 authorization code flow. Remove this stanza once all clients support proper
 # OAuth2.

cmd/lfx-mcp-server/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/knadh/koanf/providers/env/v2"
 	"github.com/knadh/koanf/v2"
 	lfxauth "github.com/linuxfoundation/lfx-mcp/internal/auth"
+	"github.com/linuxfoundation/lfx-mcp/internal/dbtsl"
 	"github.com/linuxfoundation/lfx-mcp/internal/lfxv2"
 	localOtel "github.com/linuxfoundation/lfx-mcp/internal/otel"
 	"github.com/linuxfoundation/lfx-mcp/internal/serviceapi"
@@ -61,6 +62,13 @@ type Config struct {
 	LensAPIURL            string `koanf:"lens_api_url"`
 	LensAPIAudience       string `koanf:"lens_api_audience"`
 
+	// dbt Semantic Layer configuration. Independent of the LFX API settings
+	// above: this client authenticates with a static service token and has no
+	// Auth0 dependency.
+	DBTSLHost          string `koanf:"dbt_sl_host"`
+	DBTSLEnvironmentID string `koanf:"dbt_sl_environment_id"`
+	DBTSLToken         string `koanf:"dbt_sl_token"`
+
 	// Feature flags.
 	CommitteesAsGroups bool `koanf:"committees_as_groups"`
 }
@@ -225,6 +233,9 @@ func main() {
 	f.String("onboarding_api_audience", "", "Auth0 resource server audience for the member onboarding API")
 	f.String("lens_api_url", "", "Base URL of the LFX Lens service")
 	f.String("lens_api_audience", "", "Auth0 resource server audience for the LFX Lens API")
+	f.String("dbt_sl_host", "", "dbt Semantic Layer host, e.g. tj283.semantic-layer.us1.dbt.com")
+	f.String("dbt_sl_environment_id", "", "dbt Semantic Layer environment ID")
+	f.String("dbt_sl_token", "", "dbt Semantic Layer service token")
 
 	if err := f.Parse(os.Args[1:]); err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to parse flags: %v\n", err)
@@ -475,6 +486,37 @@ func main() {
 		}
 	}
 
+	// The dbt Semantic Layer client is configured on its own, outside the LFX
+	// API block above. It authenticates with a static service token rather than
+	// an Auth0 client-credentials exchange, so nesting it under those settings
+	// would leave it unconfigured for an unrelated reason.
+	//
+	// The HTTP client is deliberately not wrapped in the serviceapi debug
+	// transport: that dumps the Authorization header, which would print this
+	// long-lived service token into the logs whenever debug traffic is on, and
+	// it currently is in production.
+	if cfg.DBTSLHost != "" && cfg.DBTSLEnvironmentID != "" && cfg.DBTSLToken != "" {
+		semanticLayerClient, err := dbtsl.NewClient(dbtsl.Config{
+			Host:          cfg.DBTSLHost,
+			EnvironmentID: cfg.DBTSLEnvironmentID,
+			Token:         cfg.DBTSLToken,
+			HTTPClient: &http.Client{
+				// Query execution polls, so a single round trip is short even
+				// when the warehouse is slow.
+				Timeout:   60 * time.Second,
+				Transport: otelhttp.NewTransport(http.DefaultTransport),
+			},
+		})
+		if err != nil {
+			logger.Warn("failed to create dbt Semantic Layer client", errKey, err)
+		} else {
+			tools.SetSemanticLayerConfig(&tools.SemanticLayerConfig{Client: semanticLayerClient})
+			logger.Info("semantic layer tools configured", "host", cfg.DBTSLHost, "environment_id", cfg.DBTSLEnvironmentID)
+		}
+	} else {
+		logger.Warn("dbt Semantic Layer not configured - semantic layer tools will return an error if called")
+	}
+
 	// Validate configuration for HTTP mode.
 	if cfg.Mode == "http" {
 		if len(cfg.MCPAPI.AuthServers) == 0 {