feat(orchestrator): enable per-build CPU/memory tracking for build sandboxes

packages/clickhouse/migrations/20260223120000_add_sandbox_type_to_host_stats.sql

@@ -0,0 +1,19 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE sandbox_host_stats_local
+    ADD COLUMN IF NOT EXISTS sandbox_type LowCardinality(String) DEFAULT 'sandbox' CODEC (ZSTD(1));
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+ALTER TABLE sandbox_host_stats
+    ADD COLUMN IF NOT EXISTS sandbox_type LowCardinality(String) DEFAULT 'sandbox' CODEC (ZSTD(1));
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE sandbox_host_stats_local DROP COLUMN IF EXISTS sandbox_type;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+ALTER TABLE sandbox_host_stats DROP COLUMN IF EXISTS sandbox_type;
+-- +goose StatementEnd

packages/clickhouse/migrations/20260223120001_add_build_columns_to_metrics_gauge.sql

@@ -0,0 +1,51 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE sandbox_metrics_gauge_local
+    ADD COLUMN IF NOT EXISTS build_id String DEFAULT '' CODEC (ZSTD(1)),
+    ADD COLUMN IF NOT EXISTS sandbox_type LowCardinality(String) DEFAULT 'sandbox' CODEC (ZSTD(1));
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+ALTER TABLE sandbox_metrics_gauge
+    ADD COLUMN IF NOT EXISTS build_id String DEFAULT '' CODEC (ZSTD(1)),
+    ADD COLUMN IF NOT EXISTS sandbox_type LowCardinality(String) DEFAULT 'sandbox' CODEC (ZSTD(1));
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+ALTER TABLE sandbox_metrics_gauge_mv MODIFY QUERY
+SELECT
+    toDateTime64(TimeUnix, 9) AS timestamp,
+    Attributes['sandbox_id'] AS sandbox_id,
+    Attributes['team_id'] AS team_id,
+    Attributes['build_id'] AS build_id,
+    Attributes['sandbox_type'] AS sandbox_type,
+    MetricName AS metric_name,
+    Value AS value
+FROM metrics_gauge
+WHERE MetricName LIKE 'e2b.sandbox.%';
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE sandbox_metrics_gauge_mv MODIFY QUERY
+SELECT
+    toDateTime64(TimeUnix, 9) AS timestamp,
+    Attributes['sandbox_id'] AS sandbox_id,
+    Attributes['team_id'] AS team_id,
+    MetricName AS metric_name,
+    Value AS value
+FROM metrics_gauge
+WHERE MetricName LIKE 'e2b.sandbox.%';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+ALTER TABLE sandbox_metrics_gauge_local
+    DROP COLUMN IF EXISTS build_id,
+    DROP COLUMN IF EXISTS sandbox_type;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+ALTER TABLE sandbox_metrics_gauge
+    DROP COLUMN IF EXISTS build_id,
+    DROP COLUMN IF EXISTS sandbox_type;
+-- +goose StatementEnd

packages/clickhouse/pkg/hoststats/delivery.go

@@ -31,9 +31,10 @@ const InsertSandboxHostStatQuery = `INSERT INTO sandbox_host_stats
     cgroup_cpu_user_usec,
     cgroup_cpu_system_usec,
     cgroup_memory_usage_bytes,
-    cgroup_memory_peak_bytes
+    cgroup_memory_peak_bytes,
+    sandbox_type
 )
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 
 type ClickhouseDelivery struct {
 	batcher *batcher.Batcher[SandboxHostStat]
@@ -123,6 +124,7 @@ func (c *ClickhouseDelivery) batchInserter(ctx context.Context, stats []SandboxH
 			stat.CgroupCPUSystemUsec,
 			stat.CgroupMemoryUsage,
 			stat.CgroupMemoryPeak,
+			stat.SandboxType,
 		)
 		if err != nil {
 			return fmt.Errorf("error appending %d host stat to batch: %w", len(stats), err)

packages/clickhouse/pkg/hoststats/hoststats.go

@@ -31,6 +31,8 @@ type SandboxHostStat struct {
 	CgroupCPUSystemUsec uint64 `ch:"cgroup_cpu_system_usec"`    // cumulative, microseconds
 	CgroupMemoryUsage   uint64 `ch:"cgroup_memory_usage_bytes"` // current, bytes
 	CgroupMemoryPeak    uint64 `ch:"cgroup_memory_peak_bytes"`  // lifetime peak, bytes
+
+	SandboxType string `ch:"sandbox_type"` // "sandbox" or "build"
 }
 
 // Delivery is the interface for delivering host stats to storage backend

packages/orchestrator/internal/metrics/sandboxes.go

@@ -179,7 +179,7 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) {
 						return err
 					}
 
-					attributes := metric.WithAttributes(attribute.String("sandbox_id", sbx.Runtime.SandboxID), attribute.String("team_id", sbx.Runtime.TeamID))
+					attributes := metric.WithAttributes(attribute.String("sandbox_id", sbx.Runtime.SandboxID), attribute.String("team_id", sbx.Runtime.TeamID), attribute.String("build_id", sbx.Runtime.BuildID), attribute.String("sandbox_type", sbx.Runtime.SandboxType.String()))
 
 					ok, err = utils.IsGTEVersion(sbx.Config.Envd.Version, minEnvVersionForMetricsTimestamp)
 					if err != nil {

packages/orchestrator/internal/sandbox/cgroup/manager.go

@@ -45,7 +45,7 @@ type Stats struct {
 // whether Start succeeded or failed). Remove() closes the memory.peak FD and
 // deletes the cgroup directory — it does not release the cgroup directory FD.
 type CgroupHandle struct {
-	sandboxID      string
+	cgroupName     string
 	path           string
 	file           *os.File // Open FD to the cgroup directory (nil after ReleaseCgroupFD)
 	memoryPeakFile *os.File // Open FD to memory.peak for per-FD reset (nil after Remove or if not available)
@@ -126,7 +126,7 @@ func (h *CgroupHandle) Remove(ctx context.Context) error {
 	}
 
 	logger.L().Debug(ctx, "removed cgroup for sandbox",
-		logger.WithSandboxID(h.sandboxID),
+		zap.String("cgroup_name", h.cgroupName),
 		zap.String("path", h.path))
 
 	return nil
@@ -141,13 +141,13 @@ func (h *CgroupHandle) Path() string {
 	return h.path
 }
 
-// SandboxID returns the sandbox ID this cgroup is for
-func (h *CgroupHandle) SandboxID() string {
+// CgroupName returns the name of the cgroup
+func (h *CgroupHandle) CgroupName() string {
 	if h == nil {
 		return ""
 	}
 
-	return h.sandboxID
+	return h.cgroupName
 }
 
 // Manager handles initialization and creation of cgroups
@@ -157,10 +157,10 @@ type Manager interface {
 	// Should be called once at orchestrator startup
 	Initialize(ctx context.Context) error
 
-	// Create creates a cgroup for a sandbox and returns a handle
+	// Create creates a cgroup with the given name and returns a handle
 	// The handle provides access to the cgroup's FD, stats, and cleanup
 	// Returns error if cgroup creation fails
-	Create(ctx context.Context, sandboxID string) (*CgroupHandle, error)
+	Create(ctx context.Context, cgroupName string) (*CgroupHandle, error)
 }
 
 type managerImpl struct{}
@@ -190,8 +190,8 @@ func (m *managerImpl) Initialize(ctx context.Context) error {
 	return nil
 }
 
-func (m *managerImpl) Create(ctx context.Context, sandboxID string) (*CgroupHandle, error) {
-	cgroupPath := m.sandboxCgroupPath(sandboxID)
+func (m *managerImpl) Create(ctx context.Context, cgroupName string) (*CgroupHandle, error) {
+	cgroupPath := m.cgroupPath(cgroupName)
 
 	if err := os.MkdirAll(cgroupPath, 0o755); err != nil {
 		return nil, fmt.Errorf("failed to create cgroup directory: %w", err)
@@ -211,22 +211,22 @@ func (m *managerImpl) Create(ctx context.Context, sandboxID string) (*CgroupHand
 	if peakErr != nil {
 		// Not fatal — memory.peak may not exist on older kernels
 		logger.L().Debug(ctx, "failed to open memory.peak",
-			logger.WithSandboxID(sandboxID),
+			zap.String("cgroup_name", cgroupName),
 			zap.String("path", memPeakPath),
 			zap.Error(peakErr))
 		memoryPeakFile = nil
 	}
 
 	handle := &CgroupHandle{
-		sandboxID:      sandboxID,
+		cgroupName:     cgroupName,
 		path:           cgroupPath,
 		file:           file,
 		memoryPeakFile: memoryPeakFile,
 		manager:        m,
 	}
 
 	logger.L().Debug(ctx, "created cgroup for sandbox",
-		logger.WithSandboxID(sandboxID),
+		zap.String("cgroup_name", cgroupName),
 		zap.String("path", cgroupPath),
 		zap.Int("fd", handle.GetFD()))
 
@@ -304,7 +304,7 @@ func (m *managerImpl) readMemoryPeak(memoryPeakFile *os.File) (uint64, error) {
 	return peakBytes, nil
 }
 
-// sandboxCgroupPath returns the filesystem path for a sandbox's cgroup
-func (m *managerImpl) sandboxCgroupPath(sandboxID string) string {
-	return filepath.Join(RootCgroupPath, fmt.Sprintf("sbx-%s", sandboxID))
+// cgroupPath returns the filesystem path for a sandbox's cgroup
+func (m *managerImpl) cgroupPath(cgroupName string) string {
+	return filepath.Join(RootCgroupPath, cgroupName)
 }

packages/orchestrator/internal/sandbox/cgroup/manager_test.go

@@ -75,7 +75,7 @@ func TestCgroupHandleLifecycle(t *testing.T) {
 	require.NotNil(t, handle)
 	defer handle.Remove(ctx)
 
-	assert.Equal(t, testSandboxID, handle.SandboxID())
+	assert.Equal(t, testSandboxID, handle.CgroupName())
 	assert.Contains(t, handle.Path(), testSandboxID)
 	assert.Positive(t, handle.GetFD())
 
@@ -141,7 +141,7 @@ func TestCgroupHandleWithProcessCreation(t *testing.T) {
 	procCgroupPath := fmt.Sprintf("/proc/%d/cgroup", cmd.Process.Pid)
 	cgroupData, err := os.ReadFile(procCgroupPath)
 	require.NoError(t, err)
-	assert.Contains(t, string(cgroupData), fmt.Sprintf("e2b/sbx-%s", testSandboxID))
+	assert.Contains(t, string(cgroupData), fmt.Sprintf("e2b/%s", testSandboxID))
 
 	cmd.Process.Kill()
 	cmd.Wait()
@@ -292,7 +292,7 @@ func TestStatsParsing(t *testing.T) {
 	t.Parallel()
 
 	tmpDir := t.TempDir()
-	cgroupPath := filepath.Join(tmpDir, "sbx-test-parse-sandbox")
+	cgroupPath := filepath.Join(tmpDir, "test-parse-sandbox")
 	err := os.MkdirAll(cgroupPath, 0o755)
 	require.NoError(t, err)
 

packages/orchestrator/internal/sandbox/fc/process.go

@@ -259,6 +259,7 @@ func (p *Process) Create(
 	hugePages bool,
 	options ProcessOptions,
 	txRateLimit TxRateLimiterConfig,
+	cgroupFD int,
 ) error {
 	ctx, childSpan := tracer.Start(ctx, "create-fc")
 	defer childSpan.End()
@@ -274,7 +275,7 @@ func (p *Process) Create(
 		sbxMetadata,
 		options.Stdout,
 		options.Stderr,
-		cgroup.NoCgroupFD,
+		cgroupFD,
 	)
 	if err != nil {
 		fcStopErr := p.Stop(ctx)

packages/orchestrator/internal/sandbox/hoststats.go

@@ -18,7 +18,6 @@ func initializeHostStatsCollector(
 	ctx context.Context,
 	sbx *Sandbox,
 	fcHandle *fc.Process,
-	buildID string,
 	runtime RuntimeMetadata,
 	config Config,
 	hostStatsDelivery hoststats.Delivery,
@@ -52,10 +51,11 @@ func initializeHostStatsCollector(
 			SandboxID:   runtime.SandboxID,
 			ExecutionID: runtime.ExecutionID,
 			TemplateID:  runtime.TemplateID,
-			BuildID:     buildID,
+			BuildID:     runtime.BuildID,
 			TeamID:      teamID,
 			VCPUCount:   config.Vcpu,
 			MemoryMB:    config.RamMB,
+			SandboxType: runtime.SandboxType,
 		},
 		int32(firecrackerPID),
 		hostStatsDelivery,

packages/orchestrator/internal/sandbox/hoststats_collector.go

@@ -39,6 +39,7 @@ type HostStatsMetadata struct {
 	TeamID      uuid.UUID
 	VCPUCount   int64
 	MemoryMB    int64
+	SandboxType SandboxType
 }
 
 func NewHostStatsCollector(
@@ -96,6 +97,7 @@ func (h *HostStatsCollector) CollectSample(ctx context.Context) error {
 		FirecrackerCPUSystemTime: times.System, // seconds
 		FirecrackerMemoryRSS:     memInfo.RSS,  // bytes
 		FirecrackerMemoryVMS:     memInfo.VMS,  // bytes
+		SandboxType:              h.metadata.SandboxType.String(),
 	}
 
 	if h.cgroupStats != nil {