Implement scalable remote control-plane coordination

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
     services:
       # PostgreSQL for comparison tests
       postgres:
-        image: postgres:16-alpine
+        image: public.ecr.aws/docker/library/postgres:16-alpine
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -62,7 +62,7 @@ jobs:
 
       # DuckLake metadata store
       ducklake-metadata:
-        image: postgres:16-alpine
+        image: public.ecr.aws/docker/library/postgres:16-alpine
         env:
           POSTGRES_USER: ducklake
           POSTGRES_PASSWORD: ducklake
@@ -138,7 +138,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:16-alpine
+        image: public.ecr.aws/docker/library/postgres:16-alpine
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -171,7 +171,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:16-alpine
+        image: public.ecr.aws/docker/library/postgres:16-alpine
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -204,10 +204,11 @@ jobs:
     timeout-minutes: 30
     env:
       DUCKGRES_KIND_CLUSTER_NAME: duckgres
+      DUCKGRES_KIND_NODE_IMAGE: kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865
 
     services:
       postgres:
-        image: postgres:16-alpine
+        image: public.ecr.aws/docker/library/postgres:16-alpine
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -237,5 +238,19 @@ jobs:
           chmod +x /tmp/kind
           sudo mv /tmp/kind /usr/local/bin/kind
           kind --version
+      - name: Clear Docker Hub credentials for kind pulls
+        run: |
+          docker logout registry-1.docker.io || true
+          docker logout docker.io || true
+          docker logout https://index.docker.io/v1/ || true
+      - name: Pre-pull kind node image
+        run: |
+          for attempt in 1 2 3; do
+            if docker pull "${DUCKGRES_KIND_NODE_IMAGE}"; then
+              exit 0
+            fi
+            sleep $((attempt * 5))
+          done
+          exit 1
       - name: Run Kubernetes integration tests
         run: just test-k8s-integration

README.md

@@ -199,6 +199,7 @@ Run with config file:
 | `DUCKGRES_THREADS` | DuckDB threads per session | `runtime.NumCPU()` |
 | `DUCKGRES_PROCESS_ISOLATION` | Enable process isolation (`1` or `true`) | `false` |
 | `DUCKGRES_IDLE_TIMEOUT` | Connection idle timeout (e.g., `30m`, `1h`, `-1` to disable) | `24h` |
+| `DUCKGRES_HANDOVER_DRAIN_TIMEOUT` | Max time to drain planned shutdowns and upgrades before forcing exit | `24h` in process mode, `15m` in remote K8s mode |
 | `DUCKGRES_K8S_SHARED_WARM_TARGET` | Neutral shared warm-worker target for K8s multi-tenant mode (`0` disables prewarm) | `0` |
 | `DUCKGRES_DUCKLAKE_METADATA_STORE` | DuckLake metadata connection string | - |
 | `POSTHOG_API_KEY` | PostHog project API key (`phc_...`); enables log export | - |
@@ -598,7 +599,7 @@ kill -USR2 <control-plane-pid>
 
 ### Remote Worker Backend
 
-In Kubernetes environments, `--worker-backend remote` is now the multitenant path only. It requires `--config-store`, and the control plane then spawns worker pods via the Kubernetes API, communicates with them over gRPC (Arrow Flight SQL), and uses owner references for automatic garbage collection when the control plane pod is deleted.
+In Kubernetes environments, `--worker-backend remote` is the multitenant path. It requires `--config-store`. Control-plane replicas coordinate through durable runtime rows in the config-store Postgres DB, spawn worker pods via the Kubernetes API, and communicate with them over gRPC (Arrow Flight SQL). Planned rolling deploys mark old replicas draining, fail readiness, and wait up to `handover_drain_timeout` before forcing shutdown. Unplanned control-plane failure still drops live pgwire connections; Flight may reconnect with a durable session token if the worker survives and the token is still valid.
 
 When a shared warm-worker target is configured (`--k8s-shared-warm-target`), the pool keeps workers neutral at startup, reserves them per org, activates tenant runtime over the activation RPC, and retires them after use. The full lifecycle is: idle → reserved → activating → hot → draining → retired.
 

controlplane/configstore/models.go

@@ -186,6 +186,92 @@ type QueryLogConfig struct {
 
 func (QueryLogConfig) TableName() string { return "duckgres_query_log_config" }
 
+// ControlPlaneInstanceState describes the liveness state of a control-plane instance.
+type ControlPlaneInstanceState string
+
+const (
+	ControlPlaneInstanceStateActive   ControlPlaneInstanceState = "active"
+	ControlPlaneInstanceStateDraining ControlPlaneInstanceState = "draining"
+	ControlPlaneInstanceStateExpired  ControlPlaneInstanceState = "expired"
+)
+
+// ControlPlaneInstance is a runtime coordination record for one control-plane process.
+// These rows live in the runtime schema, not the snapshot-backed config tables.
+type ControlPlaneInstance struct {
+	ID              string                    `gorm:"primaryKey;size:255" json:"id"`
+	PodName         string                    `gorm:"size:255;not null" json:"pod_name"`
+	PodUID          string                    `gorm:"size:255;not null" json:"pod_uid"`
+	BootID          string                    `gorm:"size:255;not null" json:"boot_id"`
+	State           ControlPlaneInstanceState `gorm:"size:32;not null" json:"state"`
+	StartedAt       time.Time                 `json:"started_at"`
+	LastHeartbeatAt time.Time                 `gorm:"index" json:"last_heartbeat_at"`
+	DrainingAt      *time.Time                `json:"draining_at,omitempty"`
+	ExpiredAt       *time.Time                `json:"expired_at,omitempty"`
+	CreatedAt       time.Time                 `json:"created_at"`
+	UpdatedAt       time.Time                 `json:"updated_at"`
+}
+
+func (ControlPlaneInstance) TableName() string { return "cp_instances" }
+
+// WorkerState is the durable lifecycle state for a worker pod.
+type WorkerState string
+
+const (
+	WorkerStateSpawning   WorkerState = "spawning"
+	WorkerStateIdle       WorkerState = "idle"
+	WorkerStateReserved   WorkerState = "reserved"
+	WorkerStateActivating WorkerState = "activating"
+	WorkerStateHot        WorkerState = "hot"
+	WorkerStateDraining   WorkerState = "draining"
+	WorkerStateRetired    WorkerState = "retired"
+	WorkerStateLost       WorkerState = "lost"
+)
+
+// WorkerRecord is the durable runtime coordination record for one worker pod.
+type WorkerRecord struct {
+	WorkerID            int         `gorm:"primaryKey" json:"worker_id"`
+	PodName             string      `gorm:"size:255;not null;uniqueIndex" json:"pod_name"`
+	PodUID              string      `gorm:"size:255" json:"pod_uid"`
+	State               WorkerState `gorm:"size:32;not null;index" json:"state"`
+	OrgID               string      `gorm:"size:255;index" json:"org_id"`
+	OwnerCPInstanceID   string      `gorm:"size:255;index" json:"owner_cp_instance_id"`
+	OwnerEpoch          int64       `gorm:"not null" json:"owner_epoch"`
+	ActivationStartedAt *time.Time  `json:"activation_started_at,omitempty"`
+	LastHeartbeatAt     time.Time   `json:"last_heartbeat_at"`
+	RetireReason        string      `gorm:"size:64" json:"retire_reason"`
+	CreatedAt           time.Time   `json:"created_at"`
+	UpdatedAt           time.Time   `json:"updated_at"`
+}
+
+func (WorkerRecord) TableName() string { return "worker_records" }
+
+// FlightSessionState is the durable reconnect state for Flight-only sessions.
+type FlightSessionState string
+
+const (
+	FlightSessionStateActive       FlightSessionState = "active"
+	FlightSessionStateReconnecting FlightSessionState = "reconnecting"
+	FlightSessionStateExpired      FlightSessionState = "expired"
+	FlightSessionStateClosed       FlightSessionState = "closed"
+)
+
+// FlightSessionRecord is the durable reconnect record for Flight sessions.
+type FlightSessionRecord struct {
+	SessionToken string             `gorm:"primaryKey;size:255" json:"session_token"`
+	Username     string             `gorm:"size:255;not null" json:"username"`
+	OrgID        string             `gorm:"size:255;not null" json:"org_id"`
+	WorkerID     int                `gorm:"not null;index" json:"worker_id"`
+	OwnerEpoch   int64              `gorm:"not null" json:"owner_epoch"`
+	CPInstanceID string             `gorm:"size:255" json:"cp_instance_id"`
+	State        FlightSessionState `gorm:"size:32;not null" json:"state"`
+	ExpiresAt    time.Time          `gorm:"index" json:"expires_at"`
+	LastSeenAt   time.Time          `json:"last_seen_at"`
+	CreatedAt    time.Time          `json:"created_at"`
+	UpdatedAt    time.Time          `json:"updated_at"`
+}
+
+func (FlightSessionRecord) TableName() string { return "flight_session_records" }
+
 // OrgConfig is a convenience view combining org metadata with resource limits.
 type OrgConfig struct {
 	Name         string