fix k3s shutdown cleanup

internal/stack/backend_k3s.go

@@ -20,6 +20,7 @@ const (
 	k3sConfigFile = "k3s-config.yaml"
 	k3sPidFile    = ".k3s.pid"
 	k3sLogFile    = "k3s.log"
+	k3sKillall    = "k3s-killall.sh"
 )
 
 // K3sBackend manages a standalone k3s cluster (bare-metal)
@@ -50,6 +51,10 @@ func (b *K3sBackend) Prerequisites(cfg *config.Config) error {
 		return fmt.Errorf("k3s not found at %s\nRun obolup.sh to install dependencies", k3sPath)
 	}
 
+	if _, err := b.killallPath(cfg); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -213,55 +218,44 @@ func (b *K3sBackend) Up(cfg *config.Config, u *ui.UI, stackID string) ([]byte, e
 func (b *K3sBackend) Down(cfg *config.Config, u *ui.UI, stackID string) error {
 	pid, err := b.readPid(cfg)
 	if err != nil {
-		u.Warn("k3s PID file not found, may not be running")
-		return nil //nolint:nilerr // missing PID file means nothing to stop
-	}
+		u.Warn("k3s PID file not found; cleaning up any orphaned runtime state")
+	} else if !b.isProcessAlive(pid) {
+		u.Warn("k3s process not running; cleaning up orphaned runtime state")
+	} else {
+		u.Infof("Stopping k3s (pid: %d)", pid)
 
-	if !b.isProcessAlive(pid) {
-		u.Warn("k3s process not running, cleaning up PID file")
-		b.removePidFile(cfg)
+		pidStr := strconv.Itoa(pid)
 
-		return nil
-	}
-
-	u.Infof("Stopping k3s (pid: %d)", pid)
+		stopCmd := exec.Command("sudo", "kill", "-TERM", pidStr)
+		if err := u.Exec(ui.ExecConfig{
+			Name: "Sending SIGTERM to k3s",
+			Cmd:  stopCmd,
+		}); err != nil {
+			u.Warnf("SIGTERM failed: %v", err)
+		}
 
-	pidStr := strconv.Itoa(pid)
+		// Give k3s a chance to shut down gracefully before the official cleanup
+		// helper stops any remaining containers and runtime processes.
+		deadline := time.Now().Add(30 * time.Second)
+		for time.Now().Before(deadline) {
+			if !b.isProcessAlive(pid) {
+				break
+			}
 
-	stopCmd := exec.Command("sudo", "kill", "-TERM", pidStr)
-	if err := u.Exec(ui.ExecConfig{
-		Name: "Sending SIGTERM to k3s",
-		Cmd:  stopCmd,
-	}); err != nil {
-		u.Warnf("SIGTERM failed, sending SIGKILL: %v", err)
+			time.Sleep(1 * time.Second)
+		}
 
-		_ = exec.Command("sudo", "kill", "-9", pidStr).Run()
-	}
+		if b.isProcessAlive(pid) {
+			u.Warn("k3s did not exit after 30s; sending SIGKILL")
 
-	// Wait for process to exit
-	deadline := time.Now().Add(30 * time.Second)
-	for time.Now().Before(deadline) {
-		if !b.isProcessAlive(pid) {
-			break
+			if err := exec.Command("sudo", "kill", "-KILL", pidStr).Run(); err != nil {
+				u.Warnf("SIGKILL failed: %v", err)
+			}
 		}
-
-		time.Sleep(1 * time.Second)
 	}
 
-	// Clean up orphaned k3s child processes
-	killallPath := "/usr/local/bin/k3s-killall.sh"
-	if _, err := os.Stat(killallPath); err == nil {
-		cleanCmd := exec.Command("sudo", killallPath)
-		_ = u.Exec(ui.ExecConfig{
-			Name: "Running k3s cleanup",
-			Cmd:  cleanCmd,
-		})
-	} else {
-		_ = exec.Command("sudo", "pkill", "-TERM", "-f", "containerd-shim.*k3s").Run()
-
-		time.Sleep(2 * time.Second)
-
-		_ = exec.Command("sudo", "pkill", "-KILL", "-f", "containerd-shim.*k3s").Run()
+	if err := b.cleanupRuntime(cfg, u); err != nil {
+		return err
 	}
 
 	b.removePidFile(cfg)
@@ -271,40 +265,86 @@ func (b *K3sBackend) Down(cfg *config.Config, u *ui.UI, stackID string) error {
 }
 
 func (b *K3sBackend) Destroy(cfg *config.Config, u *ui.UI, stackID string) error {
-	// Stop if running
-	_ = b.Down(cfg, u, stackID)
+	// K3s has no separate cluster container to delete. Fully stop its runtime,
+	// but leave persistent data to stack.Purge, which only deletes cfg.DataDir
+	// when the operator explicitly passes --force. Never run the system-wide
+	// K3s uninstaller: the binary and service may be shared with another stack.
+	return b.Down(cfg, u, stackID)
+}
 
-	// Clean up k3s state directories
+func (b *K3sBackend) DataDir(cfg *config.Config) string {
 	absDataDir, _ := filepath.Abs(cfg.DataDir)
+	return absDataDir
+}
 
-	cleanDirs := []string{
-		"/var/lib/rancher/k3s",
-		"/etc/rancher/k3s",
-		filepath.Join(absDataDir, "k3s"),
-	}
-	for _, dir := range cleanDirs {
-		if _, err := os.Stat(dir); err == nil {
-			u.Dim("  Cleaning up: " + dir)
-			_ = exec.Command("sudo", "rm", "-rf", dir).Run()
+// killallPath locates the official cleanup helper generated by the K3s
+// installer. INSTALL_K3S_BIN_DIR puts it beside the k3s binary, including the
+// .workspace/bin layout used by Obol development and backend tests.
+func (b *K3sBackend) killallPath(cfg *config.Config) (string, error) {
+	candidates := []string{filepath.Join(cfg.BinDir, k3sKillall)}
+
+	// If cfg.BinDir/k3s is a symlink into a system installation, also check
+	// beside its resolved target before the standard installer locations.
+	if resolvedK3s, err := filepath.EvalSymlinks(filepath.Join(cfg.BinDir, "k3s")); err == nil {
+		resolvedKillall := filepath.Join(filepath.Dir(resolvedK3s), k3sKillall)
+		if resolvedKillall != candidates[0] {
+			candidates = append(candidates, resolvedKillall)
 		}
 	}
 
-	// Run uninstall script if available
-	uninstallPath := "/usr/local/bin/k3s-uninstall.sh"
-	if _, err := os.Stat(uninstallPath); err == nil {
-		uninstallCmd := exec.Command("sudo", uninstallPath)
-		_ = u.Exec(ui.ExecConfig{
-			Name: "Running k3s uninstall",
-			Cmd:  uninstallCmd,
-		})
+	candidates = append(candidates,
+		"/usr/local/bin/k3s-killall.sh",
+		"/opt/bin/k3s-killall.sh",
+	)
+	if path, ok := firstExecutableFile(candidates); ok {
+		return path, nil
 	}
 
-	return nil
+	return "", fmt.Errorf(
+		"executable %s not found for k3s at %s; reinstall k3s with its official installer so the cleanup helper is available",
+		k3sKillall,
+		filepath.Join(cfg.BinDir, "k3s"),
+	)
 }
 
-func (b *K3sBackend) DataDir(cfg *config.Config) string {
-	absDataDir, _ := filepath.Abs(cfg.DataDir)
-	return absDataDir
+func firstExecutableFile(candidates []string) (string, bool) {
+	for _, candidate := range candidates {
+		info, err := os.Stat(candidate)
+		if err == nil && !info.IsDir() && info.Mode().Perm()&0o111 != 0 {
+			return candidate, true
+		}
+	}
+
+	return "", false
+}
+
+// cleanupRuntime uses K3s's version-matched cleanup helper instead of trying
+// to duplicate its containerd, CNI, mount, and iptables cleanup in Obol.
+func (b *K3sBackend) cleanupRuntime(cfg *config.Config, u *ui.UI) error {
+	killallPath, err := b.killallPath(cfg)
+	if err != nil {
+		return err
+	}
+
+	absDataDir, err := filepath.Abs(cfg.DataDir)
+	if err != nil {
+		return fmt.Errorf("failed to get absolute path for data directory: %w", err)
+	}
+
+	cleanCmd := exec.Command(killallPath)
+	cleanCmd.Env = append(cleanCmd.Environ(), "K3S_DATA_DIR="+filepath.Join(absDataDir, "k3s"))
+
+	// The official helper re-executes itself with sudo while preserving
+	// K3S_DATA_DIR. Interactive mode keeps stdin attached if sudo must prompt.
+	if err := u.Exec(ui.ExecConfig{
+		Name:        "Running k3s cleanup",
+		Cmd:         cleanCmd,
+		Interactive: true,
+	}); err != nil {
+		return fmt.Errorf("failed to clean up k3s runtime with %s: %w", killallPath, err)
+	}
+
+	return nil
 }
 
 // readPid reads the k3s PID from the PID file