From a0992bdbe9307f0ca4ecee1c3c91860208675d62 Mon Sep 17 00:00:00 2001 From: Shreyas Kalyan Date: Sun, 14 Jun 2026 17:36:28 -0400 Subject: [PATCH] BLA-4010 add build cache operation observers --- BLACKSMITH.md | 31 +++++++- cache/disk/metrics.go | 110 ++++++++++++++++---------- cache/disk/operation_observer_test.go | 86 ++++++++++++++++++++ cache/disk/options.go | 28 +++++-- cache/metrics_context.go | 63 +++++++++++++++ cache/metrics_context_test.go | 30 +++++++ cache/s3proxy/s3proxy.go | 47 ++++++++++- cache/s3proxy/s3proxy_test.go | 70 ++++++++++++++++ utils/backendproxy/backendproxy.go | 1 + 9 files changed, 414 insertions(+), 52 deletions(-) create mode 100644 cache/disk/operation_observer_test.go create mode 100644 cache/metrics_context.go create mode 100644 cache/metrics_context_test.go diff --git a/BLACKSMITH.md b/BLACKSMITH.md index 05db9d7..1353305 100644 --- a/BLACKSMITH.md +++ b/BLACKSMITH.md @@ -33,7 +33,7 @@ other callers that do not opt in to request-scoped routing. For Bazel, FA should resolve the authorized VM/job namespace to the full physical prefix: ```text -/bazel///// +///// ``` and attach it with `cache.WithStoragePrefix`. The S3 proxy then uses that @@ -50,7 +50,7 @@ Local disk cache entries store the full request prefix as a stable hash so the LRU can distinguish identical AC/CAS digests from different repo/generation namespaces without using S3-style slash-heavy prefixes in local paths. MinIO/S3 object keys use the real request-scoped prefix directly, so broad remote -deletion still targets `/bazel/...//`. +deletion still targets `/////`. For Bazel requests, FA should also mark the request with `cache.WithRequiredStoragePrefix`. If a request reaches the S3 proxy with that @@ -70,3 +70,30 @@ tags, and security advisories for `bazel-remote`. To apply an upstream patch: 5. Run the FA agent build and Buck2 cache tests before merging. BLA-4006 should make CAS namespacing changes in this repository. + +## Build cache operation observation + +BLA-4010 adds optional cache operation observation for FA-owned customer +metrics. Callers may attach opaque identity labels with +`cache.WithMetricsLabels`. bazel-remote stores and forwards those labels but +does not interpret tenant, repository, VM, or job identity. + +The disk cache accepts an optional `cache.OperationObserver` and invokes it next +to the existing endpoint metrics decorator for semantic cache outcomes: + +- `action_cache_get`: `hit`, `miss`, or `error` +- `cas_lookup`: `hit`, `miss`, or `error` + +The S3 proxy accepts the same observer and records backend async upload health +only: + +- `backend_upload`: `error` or `dropped` + +Client transfer bytes are intentionally not inferred inside bazel-remote; FA +observes gRPC request/response payloads and emits `client_upload` and +`client_download` rows with byte counts. + +Nil observers preserve existing behavior. Observer panics are swallowed through +the cache package helper so metrics collection cannot change cache request +outcomes. The fork still has no Laravel/Web dependency; FA owns aggregation and +ClickHouse delivery. diff --git a/cache/disk/metrics.go b/cache/disk/metrics.go index 831f0ed..253bf1e 100644 --- a/cache/disk/metrics.go +++ b/cache/disk/metrics.go @@ -12,16 +12,20 @@ import ( ) type metricsDecorator struct { - counter *prometheus.CounterVec + counter *prometheus.CounterVec + observer cache.OperationObserver *diskCache } const ( - hitStatus = "hit" - missStatus = "miss" + hitStatus = "hit" + missStatus = "miss" + errorStatus = "error" containsMethod = "contains" getMethod = "get" + actionCacheGet = "action_cache_get" + casLookup = "cas_lookup" //putMethod = "put" acKind = "ac" // This must be lowercase to match cache.EntryKind.String() @@ -30,23 +34,25 @@ const ( ) func (m *metricsDecorator) RegisterMetrics() { - prometheus.MustRegister(m.counter) + if m.counter != nil { + prometheus.MustRegister(m.counter) + } m.diskCache.RegisterMetrics() } func (m *metricsDecorator) Get(ctx context.Context, kind cache.EntryKind, hash string, size int64, offset int64) (io.ReadCloser, int64, error) { rc, size, err := m.diskCache.Get(ctx, kind, hash, size, offset) if err != nil { + m.recordLookup(ctx, kind, getMethod, errorStatus, "get_failed", 1, 0) return rc, size, err } - lbls := prometheus.Labels{"method": getMethod, "kind": kind.String()} + status := missStatus if rc != nil { - lbls["status"] = hitStatus - } else { - lbls["status"] = missStatus + status = hitStatus } - m.counter.With(lbls).Inc() + m.incCounter(getMethod, kind.String(), status, 1) + m.recordLookup(ctx, kind, getMethod, status, "", 1, nonNegativeUint64(size)) return rc, size, nil } @@ -54,16 +60,16 @@ func (m *metricsDecorator) Get(ctx context.Context, kind cache.EntryKind, hash s func (m *metricsDecorator) GetValidatedActionResult(ctx context.Context, hash string) (*pb.ActionResult, []byte, error) { ar, data, err := m.diskCache.GetValidatedActionResult(ctx, hash) if err != nil { + m.record(ctx, actionCacheGet, errorStatus, "get_action_result_failed", 1, 0) return ar, data, err } - lbls := prometheus.Labels{"method": getMethod, "kind": acKind} + status := missStatus if ar != nil { - lbls["status"] = hitStatus - } else { - lbls["status"] = missStatus + status = hitStatus } - m.counter.With(lbls).Inc() + m.incCounter(getMethod, acKind, status, 1) + m.record(ctx, actionCacheGet, status, "", 1, uint64(len(data))) return ar, data, err } @@ -71,19 +77,16 @@ func (m *metricsDecorator) GetValidatedActionResult(ctx context.Context, hash st func (m *metricsDecorator) GetZstd(ctx context.Context, hash string, size int64, offset int64) (io.ReadCloser, int64, error) { rc, size, err := m.diskCache.GetZstd(ctx, hash, size, offset) if err != nil { + m.record(ctx, casLookup, errorStatus, "get_zstd_failed", 1, 0) return rc, size, err } - lbls := prometheus.Labels{ - "method": getMethod, - "kind": "cas", - } + status := missStatus if rc != nil { - lbls["status"] = hitStatus - } else { - lbls["status"] = missStatus + status = hitStatus } - m.counter.With(lbls).Inc() + m.incCounter(getMethod, casKind, status, 1) + m.record(ctx, casLookup, status, "", 1, nonNegativeUint64(size)) return rc, size, nil } @@ -91,13 +94,12 @@ func (m *metricsDecorator) GetZstd(ctx context.Context, hash string, size int64, func (m *metricsDecorator) Contains(ctx context.Context, kind cache.EntryKind, hash string, size int64) (bool, int64) { ok, size := m.diskCache.Contains(ctx, kind, hash, size) - lbls := prometheus.Labels{"method": containsMethod, "kind": kind.String()} + status := missStatus if ok { - lbls["status"] = hitStatus - } else { - lbls["status"] = missStatus + status = hitStatus } - m.counter.With(lbls).Inc() + m.incCounter(containsMethod, kind.String(), status, 1) + m.recordLookup(ctx, kind, containsMethod, status, "", 1, nonNegativeUint64(size)) return ok, size } @@ -106,6 +108,7 @@ func (m *metricsDecorator) FindMissingCasBlobs(ctx context.Context, blobs []*pb. numLooking := len(blobs) digests, err := m.diskCache.FindMissingCasBlobs(ctx, blobs) if err != nil { + m.record(ctx, casLookup, errorStatus, "find_missing_cas_blobs_failed", uint64(numLooking), 0) return digests, err } @@ -113,22 +116,49 @@ func (m *metricsDecorator) FindMissingCasBlobs(ctx context.Context, blobs []*pb. numFound := numLooking - numMissing - hitLabels := prometheus.Labels{ - "method": containsMethod, - "kind": "cas", - "status": hitStatus, + m.incCounter(containsMethod, casKind, hitStatus, float64(numFound)) + m.incCounter(containsMethod, casKind, missStatus, float64(numMissing)) + if numFound > 0 { + m.record(ctx, casLookup, hitStatus, "", uint64(numFound), 0) + } + if numMissing > 0 { + m.record(ctx, casLookup, missStatus, "", uint64(numMissing), 0) } - hits := m.counter.With(hitLabels) - missLabels := prometheus.Labels{ - "method": containsMethod, - "kind": "cas", - "status": missStatus, + return digests, nil +} + +func (m *metricsDecorator) incCounter(method, kind, status string, value float64) { + if m.counter == nil || value == 0 { + return } - misses := m.counter.With(missLabels) + m.counter.With(prometheus.Labels{"method": method, "kind": kind, "status": status}).Add(value) +} - hits.Add(float64(numFound)) - misses.Add(float64(numMissing)) +func (m *metricsDecorator) recordLookup(ctx context.Context, kind cache.EntryKind, method, status, reason string, ops uint64, bytes uint64) { + switch kind { + case cache.AC: + m.record(ctx, actionCacheGet, status, reason, ops, bytes) + case cache.CAS: + m.record(ctx, casLookup, status, reason, ops, bytes) + default: + m.record(ctx, method, status, reason, ops, bytes) + } +} - return digests, nil +func (m *metricsDecorator) record(ctx context.Context, operation, status, reason string, ops uint64, bytes uint64) { + cache.ObserveOperation(ctx, m.observer, cache.OperationOutcome{ + Method: operation, + Status: status, + Reason: reason, + Ops: ops, + Bytes: bytes, + }) +} + +func nonNegativeUint64(value int64) uint64 { + if value < 0 { + return 0 + } + return uint64(value) } diff --git a/cache/disk/operation_observer_test.go b/cache/disk/operation_observer_test.go new file mode 100644 index 0000000..4504f85 --- /dev/null +++ b/cache/disk/operation_observer_test.go @@ -0,0 +1,86 @@ +package disk + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + stdlog "log" + "testing" + + "github.com/buchgr/bazel-remote/v2/cache" + pb "github.com/buchgr/bazel-remote/v2/genproto/build/bazel/remote/execution/v2" + "google.golang.org/protobuf/proto" +) + +type recordingObserver struct { + outcomes []cache.OperationOutcome +} + +func (r *recordingObserver) RecordOutcome(_ context.Context, outcome cache.OperationOutcome) { + r.outcomes = append(r.outcomes, outcome) +} + +func TestOperationObserverReceivesActionCacheHitAndMiss(t *testing.T) { + observer := &recordingObserver{} + diskCache, err := New(t.TempDir(), 1024*1024, WithOperationObserver(observer), WithAccessLogger(stdlog.New(io.Discard, "", 0))) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + result := &pb.ActionResult{StdoutRaw: []byte("ok")} + data, err := proto.Marshal(result) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + sum := sha256.Sum256(data) + hash := hex.EncodeToString(sum[:]) + if err := diskCache.Put(context.Background(), cache.AC, hash, int64(len(data)), bytes.NewReader(data)); err != nil { + t.Fatalf("Put() error = %v", err) + } + if _, _, err := diskCache.GetValidatedActionResult(context.Background(), hash); err != nil { + t.Fatalf("GetValidatedActionResult(hit) error = %v", err) + } + if _, _, err := diskCache.GetValidatedActionResult(context.Background(), "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"); err != nil { + t.Fatalf("GetValidatedActionResult(miss) error = %v", err) + } + + requireOutcome(t, observer.outcomes, actionCacheGet, hitStatus) + requireOutcome(t, observer.outcomes, actionCacheGet, missStatus) +} + +func TestOperationObserverReceivesFindMissingCasBlobsOutcomes(t *testing.T) { + observer := &recordingObserver{} + diskCache, err := New(t.TempDir(), 1024*1024, WithOperationObserver(observer), WithAccessLogger(stdlog.New(io.Discard, "", 0))) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + data := []byte("blob") + sum := sha256.Sum256(data) + hash := hex.EncodeToString(sum[:]) + if err := diskCache.Put(context.Background(), cache.CAS, hash, int64(len(data)), bytes.NewReader(data)); err != nil { + t.Fatalf("Put() error = %v", err) + } + _, err = diskCache.FindMissingCasBlobs(context.Background(), []*pb.Digest{ + {Hash: hash, SizeBytes: int64(len(data))}, + {Hash: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", SizeBytes: 12}, + }) + if err != nil { + t.Fatalf("FindMissingCasBlobs() error = %v", err) + } + + requireOutcome(t, observer.outcomes, casLookup, hitStatus) + requireOutcome(t, observer.outcomes, casLookup, missStatus) +} + +func requireOutcome(t *testing.T, outcomes []cache.OperationOutcome, method string, status string) { + t.Helper() + for _, outcome := range outcomes { + if outcome.Method == method && outcome.Status == status { + return + } + } + t.Fatalf("missing outcome method=%s status=%s in %+v", method, status, outcomes) +} diff --git a/cache/disk/options.go b/cache/disk/options.go index 9d28656..4c41cb0 100644 --- a/cache/disk/options.go +++ b/cache/disk/options.go @@ -86,17 +86,18 @@ func WithAccessLogger(logger *log.Logger) Option { func WithEndpointMetrics() Option { return func(c *CacheConfig) error { - if c.metrics != nil { + if c.metrics != nil && c.metrics.counter != nil { return fmt.Errorf("WithEndpointMetrics specified multiple times") } - c.metrics = &metricsDecorator{ - counter: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "bazel_remote_incoming_requests_total", - Help: "The number of incoming cache requests", - }, - []string{"method", "kind", "status"}), + if c.metrics == nil { + c.metrics = &metricsDecorator{} } + c.metrics.counter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "bazel_remote_incoming_requests_total", + Help: "The number of incoming cache requests", + }, + []string{"method", "kind", "status"}) c.metrics.counter.WithLabelValues("get", "cas", "hit").Add(0) c.metrics.counter.WithLabelValues("get", "cas", "miss").Add(0) @@ -108,3 +109,16 @@ func WithEndpointMetrics() Option { return nil } } + +func WithOperationObserver(observer cache.OperationObserver) Option { + return func(c *CacheConfig) error { + if observer == nil { + return nil + } + if c.metrics == nil { + c.metrics = &metricsDecorator{} + } + c.metrics.observer = observer + return nil + } +} diff --git a/cache/metrics_context.go b/cache/metrics_context.go new file mode 100644 index 0000000..ddd44e6 --- /dev/null +++ b/cache/metrics_context.go @@ -0,0 +1,63 @@ +package cache + +import "context" + +type metricsLabelsContextKey struct{} + +// MetricsLabels contains opaque caller-supplied dimensions for cache +// operation observers. bazel-remote treats these values as labels only and +// does not interpret tenant or job identity. +type MetricsLabels struct { + InstallationID string + RepositoryID string + Generation string + BuildToolID string + VMID string + JobID string + RunID string +} + +// WithMetricsLabels attaches caller-owned metrics labels to a request context. +func WithMetricsLabels(ctx context.Context, labels MetricsLabels) context.Context { + return context.WithValue(ctx, metricsLabelsContextKey{}, labels) +} + +// MetricsLabelsFromContext returns caller-owned metrics labels, if present. +func MetricsLabelsFromContext(ctx context.Context) (MetricsLabels, bool) { + labels, ok := ctx.Value(metricsLabelsContextKey{}).(MetricsLabels) + return labels, ok +} + +// OperationObserver receives best-effort cache operation outcomes. Observer +// implementations must not affect cache request behavior. +type OperationObserver interface { + RecordOutcome(ctx context.Context, outcome OperationOutcome) +} + +// OperationOutcome describes an observed cache operation outcome. +type OperationOutcome struct { + Labels MetricsLabels + Kind EntryKind + HasKind bool + Method string + Status string + Reason string + Ops uint64 + Bytes uint64 +} + +// ObserveOperation records an outcome when an observer is configured. Panics +// from observer implementations are swallowed so metrics can never change +// cache request behavior. +func ObserveOperation(ctx context.Context, observer OperationObserver, outcome OperationOutcome) { + if observer == nil { + return + } + if labels, ok := MetricsLabelsFromContext(ctx); ok { + outcome.Labels = labels + } + defer func() { + _ = recover() + }() + observer.RecordOutcome(ctx, outcome) +} diff --git a/cache/metrics_context_test.go b/cache/metrics_context_test.go new file mode 100644 index 0000000..a4a0d2a --- /dev/null +++ b/cache/metrics_context_test.go @@ -0,0 +1,30 @@ +package cache + +import ( + "context" + "testing" +) + +func TestMetricsLabelsRoundTrip(t *testing.T) { + labels := MetricsLabels{ + InstallationID: "10", + RepositoryID: "717982840", + Generation: "v0", + BuildToolID: "bazel", + VMID: "vm-123", + JobID: "job-456", + RunID: "run-789", + } + + got, ok := MetricsLabelsFromContext(WithMetricsLabels(context.Background(), labels)) + if !ok { + t.Fatal("MetricsLabelsFromContext ok = false, want true") + } + if got != labels { + t.Fatalf("MetricsLabelsFromContext = %+v, want %+v", got, labels) + } +} + +func TestObserveOperationWithoutObserverDoesNothing(t *testing.T) { + ObserveOperation(context.Background(), nil, OperationOutcome{Method: "get", Status: "hit"}) +} diff --git a/cache/s3proxy/s3proxy.go b/cache/s3proxy/s3proxy.go index 23ea0e4..de9218b 100644 --- a/cache/s3proxy/s3proxy.go +++ b/cache/s3proxy/s3proxy.go @@ -41,6 +41,15 @@ type s3Cache struct { v2mode bool updateTimestamps bool objectKey func(prefix string, hash string, kind cache.EntryKind) string + observer cache.OperationObserver +} + +type Option func(*s3Cache) + +func WithOperationObserver(observer cache.OperationObserver) Option { + return func(c *s3Cache) { + c.observer = observer + } } var ( @@ -71,7 +80,7 @@ func New( storageMode string, accessLogger cache.Logger, errorLogger cache.Logger, numUploaders, maxQueuedUploads int, - metrics Metrics) cache.Proxy { + metrics Metrics, options ...Option) cache.Proxy { fmt.Println("Using S3 backend.") @@ -83,14 +92,14 @@ func New( } // Initialize minio client with credentials - opts := &minio.Options{ + minioOpts := &minio.Options{ Creds: Credentials, BucketLookup: BucketLookupType, Region: Region, Secure: !DisableSSL, } - minioCore, err = minio.NewCore(Endpoint, opts) + minioCore, err = minio.NewCore(Endpoint, minioOpts) if err != nil { log.Fatalln(err) } @@ -110,6 +119,9 @@ func New( v2mode: storageMode == "zstd", updateTimestamps: UpdateTimestamps, } + for _, opt := range options { + opt(c) + } if c.v2mode { c.objectKey = objectKeyV2 @@ -223,6 +235,9 @@ func (c *s3Cache) UploadFile(item backendproxy.UploadReq) { ) logResponse(c.accessLogger, "UPLOAD", c.bucket, objectKey, err) + if err != nil { + c.observeUpload(context.Background(), item, "error", "s3_put_failed") + } item.Rc.Close() } @@ -233,6 +248,7 @@ func (c *s3Cache) Put(ctx context.Context, kind cache.EntryKind, hash string, lo return } prefix, requestScopedPrefix, requirePrefix := c.prefixForContext(ctx, kind) + labels, _ := cache.MetricsLabelsFromContext(ctx) select { case c.uploadQueue <- backendproxy.UploadReq{ @@ -244,9 +260,17 @@ func (c *s3Cache) Put(ctx context.Context, kind cache.EntryKind, hash string, lo StoragePrefix: prefix, RequestScopedStoragePrefix: requestScopedPrefix, RequireStoragePrefix: requirePrefix, + MetricsLabels: labels, }: default: c.errorLogger.Printf("too many uploads queued\n") + cache.ObserveOperation(ctx, c.observer, cache.OperationOutcome{ + Method: "backend_upload", + Status: "dropped", + Reason: "upload_queue_full", + Ops: 1, + Bytes: nonNegativeUint64(logicalSize), + }) rc.Close() } } @@ -333,3 +357,20 @@ func (c *s3Cache) Contains(ctx context.Context, kind cache.EntryKind, hash strin return exists, size } + +func (c *s3Cache) observeUpload(ctx context.Context, item backendproxy.UploadReq, status string, reason string) { + cache.ObserveOperation(cache.WithMetricsLabels(ctx, item.MetricsLabels), c.observer, cache.OperationOutcome{ + Method: "backend_upload", + Status: status, + Reason: reason, + Ops: 1, + Bytes: nonNegativeUint64(item.LogicalSize), + }) +} + +func nonNegativeUint64(value int64) uint64 { + if value < 0 { + return 0 + } + return uint64(value) +} diff --git a/cache/s3proxy/s3proxy_test.go b/cache/s3proxy/s3proxy_test.go index 94587bc..8b05a0c 100644 --- a/cache/s3proxy/s3proxy_test.go +++ b/cache/s3proxy/s3proxy_test.go @@ -12,6 +12,14 @@ import ( "github.com/buchgr/bazel-remote/v2/utils/backendproxy" ) +type recordingObserver struct { + outcomes []cache.OperationOutcome +} + +func (r *recordingObserver) RecordOutcome(_ context.Context, outcome cache.OperationOutcome) { + r.outcomes = append(r.outcomes, outcome) +} + func TestObjectKey(t *testing.T) { testCases := []struct { prefix string @@ -203,6 +211,68 @@ func TestPutCapturesMissingRequiredRequestScopedPrefixForAsyncUpload(t *testing. } } +func TestPutRecordsUploadQueueDrop(t *testing.T) { + hash := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + uploadQueue := make(chan backendproxy.UploadReq, 1) + uploadQueue <- backendproxy.UploadReq{Hash: "queued", Rc: io.NopCloser(strings.NewReader("queued"))} + observer := &recordingObserver{} + var errBuf bytes.Buffer + c := &s3Cache{ + prefix: "minio-prefix/staging/10/717982840/v0/bazel", + uploadQueue: uploadQueue, + errorLogger: stdlog.New(&errBuf, "", 0), + observer: observer, + } + + ctx := cache.WithMetricsLabels(context.Background(), cache.MetricsLabels{ + InstallationID: "10", + RepositoryID: "717982840", + Generation: "v0", + BuildToolID: "bazel", + VMID: "vm-123", + JobID: "job-456", + }) + c.Put(ctx, cache.CAS, hash, 4, 4, io.NopCloser(strings.NewReader("blob"))) + + if len(observer.outcomes) != 1 { + t.Fatalf("observer outcomes len = %d, want 1", len(observer.outcomes)) + } + outcome := observer.outcomes[0] + if outcome.HasKind || outcome.Method != "backend_upload" || outcome.Status != "dropped" || outcome.Reason != "upload_queue_full" { + t.Fatalf("unexpected outcome: %+v", outcome) + } + if outcome.Labels.RepositoryID != "717982840" || outcome.Labels.JobID != "job-456" { + t.Fatalf("unexpected labels: %+v", outcome.Labels) + } +} + +func TestObserveUploadRecordsBackendUploadError(t *testing.T) { + observer := &recordingObserver{} + c := &s3Cache{observer: observer} + c.observeUpload(context.Background(), backendproxy.UploadReq{ + LogicalSize: 12, + Kind: cache.CAS, + MetricsLabels: cache.MetricsLabels{ + RepositoryID: "717982840", + JobID: "job-456", + }, + }, "error", "s3_put_failed") + + if len(observer.outcomes) != 1 { + t.Fatalf("observer outcomes len = %d, want 1", len(observer.outcomes)) + } + outcome := observer.outcomes[0] + if outcome.HasKind || outcome.Method != "backend_upload" || outcome.Status != "error" || outcome.Reason != "s3_put_failed" { + t.Fatalf("unexpected outcome: %+v", outcome) + } + if outcome.Bytes != 12 { + t.Fatalf("outcome bytes = %d, want 12", outcome.Bytes) + } + if outcome.Labels.RepositoryID != "717982840" || outcome.Labels.JobID != "job-456" { + t.Fatalf("unexpected labels: %+v", outcome.Labels) + } +} + func TestLogMissingRequiredStoragePrefix(t *testing.T) { var buf bytes.Buffer c := &s3Cache{ diff --git a/utils/backendproxy/backendproxy.go b/utils/backendproxy/backendproxy.go index f323a5d..a74d32b 100644 --- a/utils/backendproxy/backendproxy.go +++ b/utils/backendproxy/backendproxy.go @@ -18,6 +18,7 @@ type UploadReq struct { StoragePrefix string RequestScopedStoragePrefix bool RequireStoragePrefix bool + MetricsLabels cache.MetricsLabels } type Uploader interface {