From 21d683bb6d31cce0b853a31bca12b9ffc4239a82 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Mon, 10 Aug 2026 14:36:22 +0400 Subject: [PATCH] fix k3s shutdown cleanup --- internal/stack/backend_k3s.go | 168 +++++++++++++--------- internal/stack/backend_k3s_test.go | 217 +++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+), 64 deletions(-) diff --git a/internal/stack/backend_k3s.go b/internal/stack/backend_k3s.go index b536cea78..9968a663a 100644 --- a/internal/stack/backend_k3s.go +++ b/internal/stack/backend_k3s.go @@ -20,6 +20,7 @@ const ( k3sConfigFile = "k3s-config.yaml" k3sPidFile = ".k3s.pid" k3sLogFile = "k3s.log" + k3sKillall = "k3s-killall.sh" ) // K3sBackend manages a standalone k3s cluster (bare-metal) @@ -50,6 +51,10 @@ func (b *K3sBackend) Prerequisites(cfg *config.Config) error { return fmt.Errorf("k3s not found at %s\nRun obolup.sh to install dependencies", k3sPath) } + if _, err := b.killallPath(cfg); err != nil { + return err + } + return nil } @@ -213,55 +218,44 @@ func (b *K3sBackend) Up(cfg *config.Config, u *ui.UI, stackID string) ([]byte, e func (b *K3sBackend) Down(cfg *config.Config, u *ui.UI, stackID string) error { pid, err := b.readPid(cfg) if err != nil { - u.Warn("k3s PID file not found, may not be running") - return nil //nolint:nilerr // missing PID file means nothing to stop - } + u.Warn("k3s PID file not found; cleaning up any orphaned runtime state") + } else if !b.isProcessAlive(pid) { + u.Warn("k3s process not running; cleaning up orphaned runtime state") + } else { + u.Infof("Stopping k3s (pid: %d)", pid) - if !b.isProcessAlive(pid) { - u.Warn("k3s process not running, cleaning up PID file") - b.removePidFile(cfg) + pidStr := strconv.Itoa(pid) - return nil - } - - u.Infof("Stopping k3s (pid: %d)", pid) + stopCmd := exec.Command("sudo", "kill", "-TERM", pidStr) + if err := u.Exec(ui.ExecConfig{ + Name: "Sending SIGTERM to k3s", + Cmd: stopCmd, + }); err != nil { + u.Warnf("SIGTERM failed: %v", err) + } - pidStr := strconv.Itoa(pid) + // Give k3s a chance to shut down gracefully before the official cleanup + // helper stops any remaining containers and runtime processes. + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + if !b.isProcessAlive(pid) { + break + } - stopCmd := exec.Command("sudo", "kill", "-TERM", pidStr) - if err := u.Exec(ui.ExecConfig{ - Name: "Sending SIGTERM to k3s", - Cmd: stopCmd, - }); err != nil { - u.Warnf("SIGTERM failed, sending SIGKILL: %v", err) + time.Sleep(1 * time.Second) + } - _ = exec.Command("sudo", "kill", "-9", pidStr).Run() - } + if b.isProcessAlive(pid) { + u.Warn("k3s did not exit after 30s; sending SIGKILL") - // Wait for process to exit - deadline := time.Now().Add(30 * time.Second) - for time.Now().Before(deadline) { - if !b.isProcessAlive(pid) { - break + if err := exec.Command("sudo", "kill", "-KILL", pidStr).Run(); err != nil { + u.Warnf("SIGKILL failed: %v", err) + } } - - time.Sleep(1 * time.Second) } - // Clean up orphaned k3s child processes - killallPath := "/usr/local/bin/k3s-killall.sh" - if _, err := os.Stat(killallPath); err == nil { - cleanCmd := exec.Command("sudo", killallPath) - _ = u.Exec(ui.ExecConfig{ - Name: "Running k3s cleanup", - Cmd: cleanCmd, - }) - } else { - _ = exec.Command("sudo", "pkill", "-TERM", "-f", "containerd-shim.*k3s").Run() - - time.Sleep(2 * time.Second) - - _ = exec.Command("sudo", "pkill", "-KILL", "-f", "containerd-shim.*k3s").Run() + if err := b.cleanupRuntime(cfg, u); err != nil { + return err } b.removePidFile(cfg) @@ -271,40 +265,86 @@ func (b *K3sBackend) Down(cfg *config.Config, u *ui.UI, stackID string) error { } func (b *K3sBackend) Destroy(cfg *config.Config, u *ui.UI, stackID string) error { - // Stop if running - _ = b.Down(cfg, u, stackID) + // K3s has no separate cluster container to delete. Fully stop its runtime, + // but leave persistent data to stack.Purge, which only deletes cfg.DataDir + // when the operator explicitly passes --force. Never run the system-wide + // K3s uninstaller: the binary and service may be shared with another stack. + return b.Down(cfg, u, stackID) +} - // Clean up k3s state directories +func (b *K3sBackend) DataDir(cfg *config.Config) string { absDataDir, _ := filepath.Abs(cfg.DataDir) + return absDataDir +} - cleanDirs := []string{ - "/var/lib/rancher/k3s", - "/etc/rancher/k3s", - filepath.Join(absDataDir, "k3s"), - } - for _, dir := range cleanDirs { - if _, err := os.Stat(dir); err == nil { - u.Dim(" Cleaning up: " + dir) - _ = exec.Command("sudo", "rm", "-rf", dir).Run() +// killallPath locates the official cleanup helper generated by the K3s +// installer. INSTALL_K3S_BIN_DIR puts it beside the k3s binary, including the +// .workspace/bin layout used by Obol development and backend tests. +func (b *K3sBackend) killallPath(cfg *config.Config) (string, error) { + candidates := []string{filepath.Join(cfg.BinDir, k3sKillall)} + + // If cfg.BinDir/k3s is a symlink into a system installation, also check + // beside its resolved target before the standard installer locations. + if resolvedK3s, err := filepath.EvalSymlinks(filepath.Join(cfg.BinDir, "k3s")); err == nil { + resolvedKillall := filepath.Join(filepath.Dir(resolvedK3s), k3sKillall) + if resolvedKillall != candidates[0] { + candidates = append(candidates, resolvedKillall) } } - // Run uninstall script if available - uninstallPath := "/usr/local/bin/k3s-uninstall.sh" - if _, err := os.Stat(uninstallPath); err == nil { - uninstallCmd := exec.Command("sudo", uninstallPath) - _ = u.Exec(ui.ExecConfig{ - Name: "Running k3s uninstall", - Cmd: uninstallCmd, - }) + candidates = append(candidates, + "/usr/local/bin/k3s-killall.sh", + "/opt/bin/k3s-killall.sh", + ) + if path, ok := firstExecutableFile(candidates); ok { + return path, nil } - return nil + return "", fmt.Errorf( + "executable %s not found for k3s at %s; reinstall k3s with its official installer so the cleanup helper is available", + k3sKillall, + filepath.Join(cfg.BinDir, "k3s"), + ) } -func (b *K3sBackend) DataDir(cfg *config.Config) string { - absDataDir, _ := filepath.Abs(cfg.DataDir) - return absDataDir +func firstExecutableFile(candidates []string) (string, bool) { + for _, candidate := range candidates { + info, err := os.Stat(candidate) + if err == nil && !info.IsDir() && info.Mode().Perm()&0o111 != 0 { + return candidate, true + } + } + + return "", false +} + +// cleanupRuntime uses K3s's version-matched cleanup helper instead of trying +// to duplicate its containerd, CNI, mount, and iptables cleanup in Obol. +func (b *K3sBackend) cleanupRuntime(cfg *config.Config, u *ui.UI) error { + killallPath, err := b.killallPath(cfg) + if err != nil { + return err + } + + absDataDir, err := filepath.Abs(cfg.DataDir) + if err != nil { + return fmt.Errorf("failed to get absolute path for data directory: %w", err) + } + + cleanCmd := exec.Command(killallPath) + cleanCmd.Env = append(cleanCmd.Environ(), "K3S_DATA_DIR="+filepath.Join(absDataDir, "k3s")) + + // The official helper re-executes itself with sudo while preserving + // K3S_DATA_DIR. Interactive mode keeps stdin attached if sudo must prompt. + if err := u.Exec(ui.ExecConfig{ + Name: "Running k3s cleanup", + Cmd: cleanCmd, + Interactive: true, + }); err != nil { + return fmt.Errorf("failed to clean up k3s runtime with %s: %w", killallPath, err) + } + + return nil } // readPid reads the k3s PID from the PID file diff --git a/internal/stack/backend_k3s_test.go b/internal/stack/backend_k3s_test.go index c7f7bca44..2f9a0fc19 100644 --- a/internal/stack/backend_k3s_test.go +++ b/internal/stack/backend_k3s_test.go @@ -1,14 +1,29 @@ package stack import ( + "bytes" "os" "path/filepath" + "runtime" "strings" "testing" "github.com/ObolNetwork/obol-stack/internal/config" + "github.com/ObolNetwork/obol-stack/internal/ui" ) +func writeK3sTestExecutable(t *testing.T, path, body string) { + t.Helper() + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", filepath.Dir(path), err) + } + + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("WriteFile(%q): %v", path, err) + } +} + func TestK3sReadPid(t *testing.T) { tests := []struct { name string @@ -101,3 +116,205 @@ func TestK3sRemovePidFileNoop(t *testing.T) { b := &K3sBackend{} b.removePidFile(cfg) // should not panic } + +func TestK3sKillallPath(t *testing.T) { + t.Run("configured bin directory", func(t *testing.T) { + binDir := t.TempDir() + want := filepath.Join(binDir, k3sKillall) + writeK3sTestExecutable(t, want, "#!/bin/sh\nexit 0\n") + + got, err := (&K3sBackend{}).killallPath(&config.Config{BinDir: binDir}) + if err != nil { + t.Fatalf("killallPath() error: %v", err) + } + + if got != want { + t.Fatalf("killallPath() = %q, want %q", got, want) + } + }) + + t.Run("resolved k3s symlink directory", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink permissions differ on Windows") + } + + root := t.TempDir() + binDir := filepath.Join(root, "bin") + targetDir := filepath.Join(root, "installation") + k3sTarget := filepath.Join(targetDir, "k3s") + want := filepath.Join(targetDir, k3sKillall) + + writeK3sTestExecutable(t, k3sTarget, "#!/bin/sh\nexit 0\n") + writeK3sTestExecutable(t, want, "#!/bin/sh\nexit 0\n") + + want, err := filepath.EvalSymlinks(want) + if err != nil { + t.Fatalf("EvalSymlinks(%q): %v", want, err) + } + + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", binDir, err) + } + + if err := os.Symlink(k3sTarget, filepath.Join(binDir, "k3s")); err != nil { + t.Fatalf("Symlink(): %v", err) + } + + got, err := (&K3sBackend{}).killallPath(&config.Config{BinDir: binDir}) + if err != nil { + t.Fatalf("killallPath() error: %v", err) + } + + if got != want { + t.Fatalf("killallPath() = %q, want %q", got, want) + } + }) + + t.Run("configured helper must be executable", func(t *testing.T) { + binDir := t.TempDir() + path := filepath.Join(binDir, k3sKillall) + + if err := os.WriteFile(path, []byte("#!/bin/sh\nexit 0\n"), 0o600); err != nil { + t.Fatalf("WriteFile(%q): %v", path, err) + } + + if got, ok := firstExecutableFile([]string{path}); ok { + t.Fatalf("firstExecutableFile() = %q, true; want false", got) + } + }) +} + +func TestK3sCleanupRuntimeUsesConfiguredDataDir(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("K3s backend is Linux-only") + } + + root := t.TempDir() + binDir := filepath.Join(root, "bin") + marker := filepath.Join(root, "data-dir.txt") + writeK3sTestExecutable(t, filepath.Join(binDir, k3sKillall), `#!/bin/sh +printf '%s' "$K3S_DATA_DIR" > "$K3S_TEST_MARKER" +`) + t.Setenv("K3S_TEST_MARKER", marker) + + cfg := &config.Config{BinDir: binDir, DataDir: filepath.Join(root, "data")} + + var stdout, stderr bytes.Buffer + if err := (&K3sBackend{}).cleanupRuntime(cfg, ui.NewForTest(&stdout, &stderr)); err != nil { + t.Fatalf("cleanupRuntime() error: %v", err) + } + + got, err := os.ReadFile(marker) + if err != nil { + t.Fatalf("ReadFile(%q): %v", marker, err) + } + + want := filepath.Join(root, "data", "k3s") + if string(got) != want { + t.Fatalf("K3S_DATA_DIR = %q, want %q", got, want) + } +} + +func TestK3sDownCleansOrphansWithoutPidFile(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("K3s backend is Linux-only") + } + + root := t.TempDir() + binDir := filepath.Join(root, "bin") + marker := filepath.Join(root, "cleanup-ran") + writeK3sTestExecutable(t, filepath.Join(binDir, k3sKillall), `#!/bin/sh +touch "$K3S_TEST_MARKER" +`) + t.Setenv("K3S_TEST_MARKER", marker) + + cfg := &config.Config{ + BinDir: binDir, + ConfigDir: filepath.Join(root, "config"), + DataDir: filepath.Join(root, "data"), + } + + var stdout, stderr bytes.Buffer + if err := (&K3sBackend{}).Down(cfg, ui.NewForTest(&stdout, &stderr), "test-stack"); err != nil { + t.Fatalf("Down() error: %v", err) + } + + if _, err := os.Stat(marker); err != nil { + t.Fatalf("cleanup helper did not run: %v", err) + } +} + +func TestK3sDownReturnsCleanupFailure(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("K3s backend is Linux-only") + } + + root := t.TempDir() + binDir := filepath.Join(root, "bin") + writeK3sTestExecutable(t, filepath.Join(binDir, k3sKillall), "#!/bin/sh\nexit 23\n") + + cfg := &config.Config{ + BinDir: binDir, + ConfigDir: filepath.Join(root, "config"), + DataDir: filepath.Join(root, "data"), + } + if err := os.MkdirAll(cfg.ConfigDir, 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", cfg.ConfigDir, err) + } + + pidPath := filepath.Join(cfg.ConfigDir, k3sPidFile) + if err := os.WriteFile(pidPath, []byte("invalid-pid"), 0o600); err != nil { + t.Fatalf("WriteFile(%q): %v", pidPath, err) + } + + var stdout, stderr bytes.Buffer + + err := (&K3sBackend{}).Down(cfg, ui.NewForTest(&stdout, &stderr), "test-stack") + if err == nil { + t.Fatal("Down() succeeded when cleanup helper failed") + } + + if !strings.Contains(err.Error(), "failed to clean up k3s runtime") { + t.Fatalf("Down() error = %q", err) + } + + if _, err := os.Stat(pidPath); err != nil { + t.Fatalf("PID evidence was removed after cleanup failure: %v", err) + } +} + +func TestK3sDestroyPreservesPersistentData(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("K3s backend is Linux-only") + } + + root := t.TempDir() + binDir := filepath.Join(root, "bin") + writeK3sTestExecutable(t, filepath.Join(binDir, k3sKillall), "#!/bin/sh\nexit 0\n") + + cfg := &config.Config{ + BinDir: binDir, + ConfigDir: filepath.Join(root, "config"), + DataDir: filepath.Join(root, "data"), + } + + statePath := filepath.Join(cfg.DataDir, "k3s", "server", "state") + if err := os.MkdirAll(filepath.Dir(statePath), 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", filepath.Dir(statePath), err) + } + + if err := os.WriteFile(statePath, []byte("persistent"), 0o600); err != nil { + t.Fatalf("WriteFile(%q): %v", statePath, err) + } + + var stdout, stderr bytes.Buffer + if err := (&K3sBackend{}).Destroy(cfg, ui.NewForTest(&stdout, &stderr), "test-stack"); err != nil { + t.Fatalf("Destroy() error: %v", err) + } + + if got, err := os.ReadFile(statePath); err != nil { + t.Fatalf("persistent K3s data was removed: %v", err) + } else if string(got) != "persistent" { + t.Fatalf("persistent K3s data = %q, want %q", got, "persistent") + } +}