From 328a2c2d78f0f8680c3df6c6a70f5bdc73f2c26e Mon Sep 17 00:00:00 2001
From: Claudiu Belu <cbelu@cloudbasesolutions.com>
Date: Mon, 22 Jun 2026 14:32:34 +0000
Subject: [PATCH] internal/db: Remove node from dqlite cluster on failed join

dqlite.Close() tears down Raft connections but does not remove the node
from the dqlite cluster membership. When db.Join() fails after the node
has already been added to the cluster (via cli.Add in app.run), the
reverter only called Close(), leaving a ghost member in dqlite.

This ghost node causes CheckMembershipConsistency to fail (the node
exists in dqlite but not in core_cluster_members), which blocks token
generation and all future cluster joins.

Fix the Join reverter to explicitly ask the dqlite leader to remove us
before closing. Also correct a misleading comment in the Bootstrap
reverter.

Signed-off-by: Claudiu Belu <cbelu@cloudbasesolutions.com>
(cherry picked from commit f9c6754bc762d84225ca5928c5b5c5908acf1057)
---
 internal/db/dqlite.go | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/internal/db/dqlite.go b/internal/db/dqlite.go
index 2b1fe7dc..32d0e408 100644
--- a/internal/db/dqlite.go
+++ b/internal/db/dqlite.go
@@ -174,7 +174,9 @@ func (db *DqliteDB) Bootstrap(extensions types.Extensions, addr *url.URL, cluste
 	}
 
 	// Only close the dqlite app if the entire Bootstrap process fails or exits.
-	// dqlite.Close() immediately tears down all Raft connections and removes this node from the dqlite cluster.
+	// NOTE: dqlite.Close() tears down Raft connections but does NOT remove
+	// the node from the dqlite cluster. For bootstrap this is fine since
+	// the node is the sole member.
 	reverter := revert.New()
 	defer reverter.Fail()
 	reverter.Add(func() {
@@ -227,18 +229,38 @@ func (db *DqliteDB) Join(extensions types.Extensions, addr *url.URL, joinAddress
 
 	// Only close the dqlite app if the entire Join process fails or exits.
 	// This ensures dqlite.Close() is not called on every Open() retry, but only if we fully give up joining.
-	// dqlite.Close() immediately tears down all Raft connections and removes this node from the dqlite cluster.
+	//
+	// NOTE: dqlite.Close() tears down Raft connections but does NOT remove
+	// the node from the dqlite cluster. We must explicitly ask the leader to
+	// remove us before closing; otherwise the node remains as a ghost member,
+	// which can break quorum calculations and membership consistency checks.
 	reverter := revert.New()
 	defer reverter.Fail()
 	reverter.Add(func() {
-		if db.dqlite != nil {
-			closeErr := db.dqlite.Close()
-			if closeErr != nil {
-				db.log().Error("Failed to close dqlite app", slog.String("address", db.listenAddr.String()), slog.String("error", closeErr.Error()))
+		if db.dqlite == nil {
+			return
+		}
+
+		removeCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		cli, err := db.dqlite.Leader(removeCtx, dqliteClient.WithConcurrentLeaderConns(1))
+		if err != nil {
+			db.log().Warn("Failed to connect to dqlite leader to remove ourselves from the cluster", slog.String("address", db.listenAddr.String()), slog.String("error", err.Error()))
+		} else {
+			err = cli.Remove(removeCtx, db.dqlite.ID())
+			cli.Close()
+			if err != nil {
+				db.log().Warn("Failed to remove ourselves from the dqlite cluster", slog.Uint64("id", db.dqlite.ID()), slog.String("address", db.listenAddr.String()), slog.String("error", err.Error()))
 			}
+		}
 
-			db.dqlite = nil
+		closeErr := db.dqlite.Close()
+		if closeErr != nil {
+			db.log().Error("Failed to close dqlite app", slog.String("address", db.listenAddr.String()), slog.String("error", closeErr.Error()))
 		}
+
+		db.dqlite = nil
 	})
 
 	for {