diff --git a/packages/vm-agent/internal/bootstrap/bootstrap.go b/packages/vm-agent/internal/bootstrap/bootstrap.go index d3bbb325b5..cb57f67704 100644 --- a/packages/vm-agent/internal/bootstrap/bootstrap.go +++ b/packages/vm-agent/internal/bootstrap/bootstrap.go @@ -124,162 +124,8 @@ func Run(ctx context.Context, cfg *config.Config, reporter *bootlog.Reporter) er return nil } - state, err := loadState(cfg.BootstrapStatePath) - if err != nil { - return fmt.Errorf("failed to load bootstrap state: %w", err) - } - - if state != nil { - if state.WorkspaceID != cfg.WorkspaceID { - return fmt.Errorf("bootstrap state workspace mismatch: expected %s, found %s", cfg.WorkspaceID, state.WorkspaceID) - } - slog.Info("Using cached bootstrap state", "path", cfg.BootstrapStatePath) - cfg.CallbackToken = state.CallbackToken - reporter.SetToken(state.CallbackToken) - } else { - reporter.Log("bootstrap_redeem", "started", "Redeeming bootstrap credentials") - state, err = redeemBootstrapTokenWithRetry(ctx, cfg) - if err != nil { - return err - } - cfg.CallbackToken = state.CallbackToken - reporter.SetToken(state.CallbackToken) - reporter.Log("bootstrap_redeem", "completed", "Bootstrap credentials redeemed") - if err := saveState(cfg.BootstrapStatePath, state); err != nil { - return fmt.Errorf("failed to persist bootstrap state: %w", err) - } - } - - if cfg.CallbackToken == "" { - return errors.New("callback token is missing after bootstrap") - } - - // Create a named Docker volume for container-mode workspaces. - // The volume replaces the host bind-mount, eliminating permission issues. - volumeName := "" - if cfg.ContainerMode { - reporter.Log("volume_create", "started", "Creating workspace volume") - var volErr error - volumeName, volErr = ensureVolumeReady(ctx, cfg.WorkspaceID) - if volErr != nil { - reporter.Log("volume_create", "failed", "Volume creation failed", volErr.Error()) - return volErr - } - reporter.Log("volume_create", "completed", "Workspace volume ready") - } - - reporter.Log("git_clone", "started", "Cloning repository") - if err := ensureRepositoryReady(ctx, cfg, state, volumeName); err != nil { - reporter.Log("git_clone", "failed", "Repository clone failed", err.Error()) - return err - } - reporter.Log("git_clone", "completed", "Repository cloned") - - // Pre-generate credential helper on the VM host so it can be bind-mounted - // into the container. This makes git authentication available during - // devcontainer lifecycle hooks (postCreateCommand, postStartCommand, etc.). - credHelperHostPath, credErr := writeCredentialHelperToHost(cfg) - if credErr != nil { - slog.Warn("Failed to write credential helper to host (non-fatal)", "error", credErr) - reporter.Log("git_credential_helper", "failed", "Credential helper setup failed — git auth may be unavailable in lifecycle hooks", credErr.Error()) - } - bootstrapSucceeded := false - if credHelperHostPath != "" { - defer func() { - // Clean up the host-side file if bootstrap fails; on success the - // file stays for the lifetime of the workspace and is removed in - // handleDeleteWorkspace via RemoveCredentialHelperFromHost. - if !bootstrapSucceeded { - RemoveCredentialHelperFromHost(cfg.WorkspaceID) - } - }() - } - - reporter.Log("devcontainer_wait", "started", "Waiting for devcontainer CLI") - reporter.Log("devcontainer_up", "started", "Building devcontainer") - // DevcontainerConfigName is not available in the bootstrap-token path because - // bootstrapState (from redeemBootstrapToken) does not carry it. Named - // devcontainer configs are only supported via the control-plane POST /workspaces - // flow (PrepareWorkspace), which threads state.DevcontainerConfigName directly. - // The bootstrap-token path does not support caching (no DevcontainerConfigName, - // limited token context). Pass empty cacheRef to disable. - usedFallback, err := ensureDevcontainerReady(ctx, cfg, volumeName, credHelperHostPath, "", "") - if err != nil { - reporter.Log("devcontainer_up", "failed", "Devcontainer build failed", err.Error()) - return err - } - if usedFallback { - reporter.Log("devcontainer_up", "completed", "Devcontainer ready (fallback to default image)") - } else { - reporter.Log("devcontainer_up", "completed", "Devcontainer ready") - } - - // Inject apt retry config (all providers) and mirror config (provider-specific) before package installs. - // Non-fatal: if injection fails, apt will use default settings. - if containerID, findErr := findDevcontainerID(ctx, cfg); findErr == nil { - injectAptRetryConfig(ctx, containerID) - injectAptMirrorConfig(ctx, cfg, containerID) - } else { - slog.Debug("Could not find devcontainer for apt config injection (non-fatal)", "error", findErr) - } - - // Ensure gh CLI is available (install if missing from custom devcontainers). - // Non-fatal: workspace still works without gh, just can't create PRs. - reporter.Log("gh_cli", "started", "Checking GitHub CLI availability") - if err := ensureGitHubCLI(ctx, cfg); err != nil { - reporter.Log("gh_cli", "failed", "GitHub CLI install failed (non-fatal)", err.Error()) - slog.Warn("GitHub CLI install failed (non-fatal)", "error", err) - } else { - reporter.Log("gh_cli", "completed", "GitHub CLI available") - } - - reporter.Log("git_creds", "started", "Configuring git credentials") - if err := ensureGitCredentialHelper(ctx, cfg); err != nil { - reporter.Log("git_creds", "failed", "Git credential setup failed", err.Error()) - return err - } - reporter.Log("git_creds", "completed", "Git credentials configured") - - reporter.Log("git_identity", "started", "Configuring git identity") - if err := ensureGitIdentity(ctx, cfg, state); err != nil { - reporter.Log("git_identity", "failed", "Git identity setup failed", err.Error()) - return err - } - reporter.Log("git_identity", "completed", "Git identity configured") - - reporter.Log("sam_env", "started", "Configuring SAM environment") - if err := ensureSAMEnvironment(ctx, cfg, state.GitHubToken); err != nil { - reporter.Log("sam_env", "failed", "SAM environment setup failed", err.Error()) - slog.Warn("SAM environment setup failed (non-fatal)", "error", err) - } else { - reporter.Log("sam_env", "completed", "SAM environment configured") - } - - readyStatus := workspaceReadyStatusRunning - if recovery, recoveryErr := hasBuildErrorMarker(cfg); recoveryErr != nil { - slog.Warn("Failed to inspect build error marker", "workspaceID", cfg.WorkspaceID, "error", recoveryErr) - } else if recovery { - readyStatus = workspaceReadyStatusRecovery - } - if usedFallback { - readyStatus = workspaceReadyStatusRecovery - } - - // The container is fully provisioned at this point. Mark the credential - // helper as persistent so it is NOT cleaned up by the deferred function, - // even if the markWorkspaceReady callback fails (CallbackError). A - // CallbackError means the workspace is running but the control plane was - // not notified — the credential file must persist for the running container. - bootstrapSucceeded = true - - reporter.Log("workspace_ready", "started", "Marking workspace ready") - if err := markWorkspaceReady(ctx, cfg, readyStatus, ""); err != nil { - reporter.Log("workspace_ready", "failed", "Failed to mark workspace ready", err.Error()) - return &CallbackError{Err: err, Status: readyStatus} - } - reporter.Log("workspace_ready", "completed", "Workspace is ready") - - return nil + bootstrapCtx := &workspaceBootstrapContext{cfg: cfg, reporter: reporter} + return runBootstrapPlan(ctx, bootstrapTokenPlan(), bootstrapCtx) } // PrepareWorkspace provisions a workspace repository/devcontainer and configures @@ -296,169 +142,9 @@ func PrepareWorkspace(ctx context.Context, cfg *config.Config, state ProvisionSt return false, errors.New("config is required") } - bootstrap := &bootstrapState{ - WorkspaceID: cfg.WorkspaceID, - CallbackToken: cfg.CallbackToken, - GitHubToken: strings.TrimSpace(state.GitHubToken), - GitUserName: strings.TrimSpace(state.GitUserName), - GitUserEmail: strings.TrimSpace(state.GitUserEmail), - GitHubID: strings.TrimSpace(state.GitHubID), - } - - // Create a named Docker volume for container-mode workspaces. - volumeName := "" - if cfg.ContainerMode { - reporter.Log("volume_create", "started", "Creating workspace volume") - var volErr error - volumeName, volErr = ensureVolumeReady(ctx, cfg.WorkspaceID) - if volErr != nil { - reporter.Log("volume_create", "failed", "Volume creation failed", volErr.Error()) - return false, volErr - } - reporter.Log("volume_create", "completed", "Workspace volume ready") - } - - reporter.Log("git_clone", "started", "Cloning repository") - if err := ensureRepositoryReady(ctx, cfg, bootstrap, volumeName); err != nil { - reporter.Log("git_clone", "failed", "Repository clone failed", err.Error()) - return false, err - } - reporter.Log("git_clone", "completed", "Repository cloned") - - repoHasDevcontainerConfig := hasDevcontainerConfig(cfg.WorkspaceDir) - effectiveWorkspaceProfile := "" - if state.Lightweight || (state.DevcontainerConfigName == "" && !repoHasDevcontainerConfig) { - effectiveWorkspaceProfile = "lightweight" - } - - // Pre-generate credential helper on the VM host so it can be bind-mounted - // into the container during devcontainer lifecycle hooks. - credHelperHostPath, credErr := writeCredentialHelperToHost(cfg) - if credErr != nil { - slog.Warn("Failed to write credential helper to host (non-fatal)", "error", credErr) - reporter.Log("git_credential_helper", "failed", "Credential helper setup failed — git auth may be unavailable in lifecycle hooks", credErr.Error()) - } - prepareSucceeded := false - if credHelperHostPath != "" { - defer func() { - if !prepareSucceeded { - RemoveCredentialHelperFromHost(cfg.WorkspaceID) - } - }() - } - - // Resolve devcontainer cache ref (best-effort, only for non-lightweight workspaces). - cacheRef := "" - if cfg.DevcontainerCacheEnabled && !state.Lightweight && repoHasDevcontainerConfig { - var cacheErr error - cacheRef, cacheErr = prepareDevcontainerCache(ctx, cfg, bootstrap.GitHubToken, state.DevcontainerConfigName) - if cacheErr != nil { - slog.Warn("Cache registry login failed (caching disabled for this build)", "registry", cfg.DevcontainerCacheRegistry, "error", cacheErr) - cacheRef = "" - } - if cacheRef != "" { - reporter.Log("devcontainer_cache", "started", "Checking devcontainer cache") - } - } - - var usedFallback bool - var recoveryMode bool - if state.Lightweight { - // Lightweight profile: skip devcontainer build entirely, use fallback image. - // This saves 30-120 seconds by avoiding the project's .devcontainer build. - reporter.Log("devcontainer_up", "started", "Starting lightweight container (skipping devcontainer build)") - slog.Info("Lightweight mode: forcing fallback image, skipping devcontainer build", "workspaceID", cfg.WorkspaceID) - var fallbackErr error - usedFallback, fallbackErr = ensureDevcontainerFallback(ctx, cfg, volumeName, credHelperHostPath) - if fallbackErr != nil { - reporter.Log("devcontainer_up", "failed", "Lightweight container startup failed", fallbackErr.Error()) - return false, fallbackErr - } - reporter.Log("devcontainer_up", "completed", "Lightweight container ready") - } else { - reporter.Log("devcontainer_up", "started", "Building devcontainer") - var devErr error - usedFallback, devErr = ensureDevcontainerReady(ctx, cfg, volumeName, credHelperHostPath, state.DevcontainerConfigName, cacheRef) - if devErr != nil { - reporter.Log("devcontainer_up", "failed", "Devcontainer build failed", devErr.Error()) - return false, devErr - } - if usedFallback { - reporter.Log("devcontainer_up", "completed", "Devcontainer ready (fallback to default image)") - } else { - reporter.Log("devcontainer_up", "completed", "Devcontainer ready") - } - - recoveryMode = usedFallback - if markerFound, markerErr := hasBuildErrorMarker(cfg); markerErr != nil { - slog.Warn("Failed to inspect build error marker", "workspaceID", cfg.WorkspaceID, "error", markerErr) - } else if markerFound { - recoveryMode = true - } - } - - // Inject apt retry config (all providers) and mirror config (provider-specific) before package installs. - // Non-fatal: if injection fails, apt will use default settings. - if containerID, findErr := findDevcontainerID(ctx, cfg); findErr == nil { - injectAptRetryConfig(ctx, containerID) - injectAptMirrorConfig(ctx, cfg, containerID) - } else { - slog.Debug("Could not find devcontainer for apt config injection (non-fatal)", "error", findErr) - } - - // Ensure gh CLI is available (install if missing from custom devcontainers). - reporter.Log("gh_cli", "started", "Checking GitHub CLI availability") - if err := ensureGitHubCLI(ctx, cfg); err != nil { - reporter.Log("gh_cli", "failed", "GitHub CLI install failed (non-fatal)", err.Error()) - slog.Warn("GitHub CLI install failed (non-fatal)", "error", err) - } else { - reporter.Log("gh_cli", "completed", "GitHub CLI available") - } - - reporter.Log("git_creds", "started", "Configuring git credentials") - if err := ensureGitCredentialHelper(ctx, cfg); err != nil { - reporter.Log("git_creds", "failed", "Git credential setup failed", err.Error()) - return recoveryMode, err - } - reporter.Log("git_creds", "completed", "Git credentials configured") - - reporter.Log("git_identity", "started", "Configuring git identity") - if err := ensureGitIdentity(ctx, cfg, bootstrap); err != nil { - reporter.Log("git_identity", "failed", "Git identity setup failed", err.Error()) - return recoveryMode, err - } - reporter.Log("git_identity", "completed", "Git identity configured") - - reporter.Log("sam_env", "started", "Configuring SAM environment") - if err := ensureSAMEnvironment(ctx, cfg, bootstrap.GitHubToken); err != nil { - reporter.Log("sam_env", "failed", "SAM environment setup failed", err.Error()) - slog.Warn("SAM environment setup failed (non-fatal)", "error", err) - } else { - reporter.Log("sam_env", "completed", "SAM environment configured") - } - if err := ensureProjectRuntimeAssets(ctx, cfg, state.ProjectEnvVars, state.ProjectFiles); err != nil { - return recoveryMode, err - } - - // Container is fully provisioned — keep the credential helper file even if - // markWorkspaceReady fails (CallbackError means workspace is running). - prepareSucceeded = true - - reporter.Log("workspace_ready", "started", "Marking workspace ready") - readyStatus := workspaceReadyStatusRunning - if recoveryMode { - readyStatus = workspaceReadyStatusRecovery - } - if err := markWorkspaceReady(ctx, cfg, readyStatus, effectiveWorkspaceProfile); err != nil { - reporter.Log("workspace_ready", "failed", "Failed to mark workspace ready", err.Error()) - // Workspace is fully provisioned — only the callback to the control plane - // failed. Return a CallbackError so the caller can distinguish this from - // a real provisioning failure and retry the callback later. - return recoveryMode, &CallbackError{Err: err, Status: readyStatus} - } - reporter.Log("workspace_ready", "completed", "Workspace is ready") - - return recoveryMode, nil + bootstrapCtx := &workspaceBootstrapContext{cfg: cfg, reporter: reporter, provision: state} + err := runBootstrapPlan(ctx, prepareWorkspacePlan(), bootstrapCtx) + return bootstrapCtx.recoveryMode, err } func prepareDevcontainerCache(ctx context.Context, cfg *config.Config, githubToken, devcontainerConfigName string) (string, error) { @@ -920,154 +606,178 @@ func ensureDevcontainerFallback(ctx context.Context, cfg *config.Config, volumeN // it is injected into override configs as a cacheFrom source and the built image // is pushed to this ref asynchronously after a successful build. func ensureDevcontainerReady(ctx context.Context, cfg *config.Config, volumeName, credHelperHostPath, devcontainerConfigName, cacheRef string) (bool, error) { - if _, err := findDevcontainerID(ctx, cfg); err == nil { - slog.Info("Devcontainer already running", "labelKey", cfg.ContainerLabelKey, "labelValue", cfg.ContainerLabelValue) - ensureContainerUserResolved(ctx, cfg, devcontainerConfigName) - if err := ensureWorkspaceOwnership(ctx, cfg); err != nil { + if found, err := findExistingDevcontainerAndRepair(ctx, cfg, devcontainerConfigName); found || err != nil { + return false, err + } + if err := validateNamedDevcontainerConfig(cfg, devcontainerConfigName); err != nil { + return false, err + } + if err := waitForDevcontainerCLI(ctx); err != nil { + return false, err + } + + slog.Info("Starting devcontainer for workspace", "workspaceDir", cfg.WorkspaceDir) + + effectiveCacheRef := resolveEffectiveCacheRef(ctx, cacheRef) + hasConfig := hasDevcontainerConfig(cfg.WorkspaceDir) + usedFallback := false + + if hasConfig { + var err error + usedFallback, err = startRepoDevcontainerWithFallback(ctx, cfg, volumeName, credHelperHostPath, devcontainerConfigName, effectiveCacheRef) + if err != nil { return false, err } + } else if err := startDefaultDevcontainerForNoConfig(ctx, cfg, volumeName, credHelperHostPath); err != nil { + return false, err + } + + if err := ensurePostDevcontainerState(ctx, cfg, volumeName, devcontainerConfigName, usedFallback); err != nil { + return false, err + } + pushDevcontainerCacheAsync(cacheRef, usedFallback, hasConfig, cfg.ContainerLabelKey, cfg.ContainerLabelValue) + return usedFallback, nil +} + +func findExistingDevcontainerAndRepair(ctx context.Context, cfg *config.Config, devcontainerConfigName string) (bool, error) { + if _, err := findDevcontainerID(ctx, cfg); err != nil { return false, nil } + slog.Info("Devcontainer already running", "labelKey", cfg.ContainerLabelKey, "labelValue", cfg.ContainerLabelValue) + ensureContainerUserResolved(ctx, cfg, devcontainerConfigName) + if err := ensureWorkspaceOwnership(ctx, cfg); err != nil { + return true, err + } + return true, nil +} - if devcontainerConfigName != "" { - configPath := namedDevcontainerConfigPath(cfg.WorkspaceDir, devcontainerConfigName) - if _, err := os.Stat(configPath); err != nil { - if os.IsNotExist(err) { - return false, fmt.Errorf("devcontainer config %q not found at %s", devcontainerConfigName, configPath) - } - return false, fmt.Errorf("failed to inspect devcontainer config %q at %s: %w", devcontainerConfigName, configPath, err) +func validateNamedDevcontainerConfig(cfg *config.Config, devcontainerConfigName string) error { + if devcontainerConfigName == "" { + return nil + } + configPath := namedDevcontainerConfigPath(cfg.WorkspaceDir, devcontainerConfigName) + if _, err := os.Stat(configPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("devcontainer config %q not found at %s", devcontainerConfigName, configPath) } + return fmt.Errorf("failed to inspect devcontainer config %q at %s: %w", devcontainerConfigName, configPath, err) } + return nil +} - // Wait for devcontainer CLI to be available. Cloud-init installs Node.js and - // devcontainer CLI asynchronously AFTER the VM Agent starts — there is a race - // where the agent tries to run "devcontainer up" before the CLI exists. +func waitForDevcontainerCLI(ctx context.Context) error { + // Cloud-init installs Node.js and devcontainer CLI asynchronously AFTER the + // VM Agent starts, so bootstrap can race the CLI installation. if err := waitForCommand(ctx, "devcontainer"); err != nil { - return false, fmt.Errorf("devcontainer CLI never became available: %w", err) + return fmt.Errorf("devcontainer CLI never became available: %w", err) } + return nil +} - slog.Info("Starting devcontainer for workspace", "workspaceDir", cfg.WorkspaceDir) - - // Best-effort cache pull: try to pull the cached image so Docker can use - // its layers during the build. Failures are non-fatal. - cacheImagePulled := false - if cacheRef != "" { - if pullErr := cache.PullCacheImage(ctx, cacheRef); pullErr != nil { - slog.Info("No cache image available (building from scratch)", "ref", cacheRef, "reason", pullErr) - } else { - slog.Info("Cache hit: pulled devcontainer cache image", "ref", cacheRef) - cacheImagePulled = true - } +func resolveEffectiveCacheRef(ctx context.Context, cacheRef string) string { + if cacheRef == "" { + return "" } - // Only inject cacheFrom if we actually pulled the image. - effectiveCacheRef := "" - if cacheImagePulled { - effectiveCacheRef = cacheRef + if pullErr := cache.PullCacheImage(ctx, cacheRef); pullErr != nil { + slog.Info("No cache image available (building from scratch)", "ref", cacheRef, "reason", pullErr) + return "" } + slog.Info("Cache hit: pulled devcontainer cache image", "ref", cacheRef) + return cacheRef +} - hasConfig := hasDevcontainerConfig(cfg.WorkspaceDir) - usedFallback := false +func startRepoDevcontainerWithFallback(ctx context.Context, cfg *config.Config, volumeName, credHelperHostPath, devcontainerConfigName, effectiveCacheRef string) (bool, error) { + overridePath, usedFallback, err := prepareRepoDevcontainerOverride(ctx, cfg, volumeName, credHelperHostPath, devcontainerConfigName, effectiveCacheRef) + if overridePath != "" { + defer os.Remove(overridePath) + } + if err != nil { + return false, err + } + if usedFallback { + return true, nil + } + return runRepoDevcontainerUp(ctx, cfg, volumeName, credHelperHostPath, devcontainerConfigName, overridePath) +} - if hasConfig { - // Try with repo's own devcontainer config first. - // When using a volume, resolve the repo config through `devcontainer - // read-configuration` and inject workspaceMount/workspaceFolder into the - // merged config so required fields (image/dockerFile/dockerComposeFile) - // remain intact. - var overridePath string - if volumeName != "" { - var mountErr error - overridePath, mountErr = writeMountOverrideConfig(ctx, cfg, volumeName, credHelperHostPath, devcontainerConfigName, effectiveCacheRef) - if mountErr != nil { - slog.Warn("Failed to prepare repo mount override config, falling back to default image", "error", mountErr) - fallbackOutput := []byte(fmt.Sprintf("failed to prepare repo devcontainer mount override: %v\n", mountErr)) - var fallbackErr error - usedFallback, fallbackErr = fallbackToDefaultDevcontainer(ctx, cfg, volumeName, credHelperHostPath, mountErr, fallbackOutput) - if fallbackErr != nil { - return false, fallbackErr - } - } - defer os.Remove(overridePath) - } else if credHelperHostPath != "" { - // Repo has config but no volume — use a credential-only override. - var credErr error - overridePath, credErr = writeCredentialOverrideConfig(credHelperHostPath, effectiveCacheRef) - if credErr != nil { - slog.Warn("Failed to write credential override config", "error", credErr) - // Non-fatal: continue without pre-mounted credential helper. - } - if overridePath != "" { - defer os.Remove(overridePath) - } - } else if effectiveCacheRef != "" { - // No volume, no credential helper, but we have a cache ref — - // write a cache-only override config. - var cacheErr error - overridePath, cacheErr = writeCacheOnlyOverrideConfig(effectiveCacheRef) - if cacheErr != nil { - slog.Warn("Failed to write cache override config", "error", cacheErr) - } - if overridePath != "" { - defer os.Remove(overridePath) +func prepareRepoDevcontainerOverride(ctx context.Context, cfg *config.Config, volumeName, credHelperHostPath, devcontainerConfigName, effectiveCacheRef string) (string, bool, error) { + if volumeName != "" { + overridePath, mountErr := writeMountOverrideConfig(ctx, cfg, volumeName, credHelperHostPath, devcontainerConfigName, effectiveCacheRef) + if mountErr != nil { + slog.Warn("Failed to prepare repo mount override config, falling back to default image", "error", mountErr) + fallbackOutput := []byte(fmt.Sprintf("failed to prepare repo devcontainer mount override: %v\n", mountErr)) + usedFallback, fallbackErr := fallbackToDefaultDevcontainer(ctx, cfg, volumeName, credHelperHostPath, mountErr, fallbackOutput) + if fallbackErr != nil { + return "", false, fallbackErr } + return "", usedFallback, nil } - - if !usedFallback { - args := devcontainerUpArgs(cfg, overridePath, devcontainerConfigName) - if cfg.AdditionalFeatures != "" { - slog.Info("Repo has its own devcontainer config, skipping additional-features injection") - } - - buildCtx, buildCancel := devcontainerBuildContext(ctx, cfg) - cmd := exec.CommandContext(buildCtx, "devcontainer", args...) - output, err := cmd.CombinedOutput() - buildCancel() // Release timer immediately; fallback uses parent ctx. - if err != nil { - // Repo config failed — log the error and fall back to default image. - slog.Warn("Devcontainer build failed with repo config, falling back to default image", "error", err, "output", strings.TrimSpace(string(output)), "timedOut", buildCtx.Err() == context.DeadlineExceeded) - var fallbackErr error - usedFallback, fallbackErr = fallbackToDefaultDevcontainer(ctx, cfg, volumeName, credHelperHostPath, err, output) - if fallbackErr != nil { - return false, fallbackErr - } - } + return overridePath, false, nil + } + if credHelperHostPath != "" { + overridePath, credErr := writeCredentialOverrideConfig(credHelperHostPath, effectiveCacheRef) + if credErr != nil { + slog.Warn("Failed to write credential override config", "error", credErr) + return "", false, nil } - } else { - // No config — use the lightweight default image. Repos without a - // devcontainer have nothing project-specific to build, so avoid - // devcontainer Features and the slower build path entirely. - slog.Info("No repo devcontainer config found; using lightweight default image", "workspaceDir", cfg.WorkspaceDir) - buildCtx, buildCancel := devcontainerBuildContext(ctx, cfg) - _, err := runLightweightDevcontainerWithDefault(buildCtx, cfg, volumeName, credHelperHostPath) - buildCancel() - if err != nil { - return false, err + return overridePath, false, nil + } + if effectiveCacheRef != "" { + overridePath, cacheErr := writeCacheOnlyOverrideConfig(effectiveCacheRef) + if cacheErr != nil { + slog.Warn("Failed to write cache override config", "error", cacheErr) + return "", false, nil } + return overridePath, false, nil } + return "", false, nil +} - if !usedFallback { - clearBuildErrorArtifacts(ctx, cfg, volumeName) +func runRepoDevcontainerUp(ctx context.Context, cfg *config.Config, volumeName, credHelperHostPath, devcontainerConfigName, overridePath string) (bool, error) { + args := devcontainerUpArgs(cfg, overridePath, devcontainerConfigName) + if cfg.AdditionalFeatures != "" { + slog.Info("Repo has its own devcontainer config, skipping additional-features injection") } - // Best-effort async cache push: tag and push the built image in the background. - // Only push when the build succeeded with the repo's own config (not fallback). - if cacheRef != "" && !usedFallback && hasConfig { - labelKey := cfg.ContainerLabelKey - labelValue := cfg.ContainerLabelValue - go func() { - pushCtx, pushCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer pushCancel() - if pushErr := cache.PushCacheImage(pushCtx, labelKey, labelValue, cacheRef); pushErr != nil { - slog.Warn("Cache image push failed (non-fatal)", "ref", cacheRef, "error", pushErr) - } - }() + buildCtx, buildCancel := devcontainerBuildContext(ctx, cfg) + cmd := exec.CommandContext(buildCtx, "devcontainer", args...) + output, err := cmd.CombinedOutput() + buildCancel() + if err == nil { + return false, nil } + slog.Warn("Devcontainer build failed with repo config, falling back to default image", "error", err, "output", strings.TrimSpace(string(output)), "timedOut", buildCtx.Err() == context.DeadlineExceeded) + return fallbackToDefaultDevcontainer(ctx, cfg, volumeName, credHelperHostPath, err, output) +} + +func startDefaultDevcontainerForNoConfig(ctx context.Context, cfg *config.Config, volumeName, credHelperHostPath string) error { + slog.Info("No repo devcontainer config found; using lightweight default image", "workspaceDir", cfg.WorkspaceDir) + buildCtx, buildCancel := devcontainerBuildContext(ctx, cfg) + _, err := runLightweightDevcontainerWithDefault(buildCtx, cfg, volumeName, credHelperHostPath) + buildCancel() + return err +} + +func ensurePostDevcontainerState(ctx context.Context, cfg *config.Config, volumeName, devcontainerConfigName string, usedFallback bool) error { + if !usedFallback { + clearBuildErrorArtifacts(ctx, cfg, volumeName) + } ensureContainerUserResolved(ctx, cfg, devcontainerConfigName) - if err := ensureWorkspaceOwnership(ctx, cfg); err != nil { - return false, err + return ensureWorkspaceOwnership(ctx, cfg) +} + +func pushDevcontainerCacheAsync(cacheRef string, usedFallback, hasConfig bool, labelKey, labelValue string) { + if cacheRef == "" || usedFallback || !hasConfig { + return } - return usedFallback, nil + go func() { + pushCtx, pushCancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer pushCancel() + if pushErr := cache.PushCacheImage(pushCtx, labelKey, labelValue, cacheRef); pushErr != nil { + slog.Warn("Cache image push failed (non-fatal)", "ref", cacheRef, "error", pushErr) + } + }() } // devcontainerBuildContext wraps the parent context with a DevcontainerBuildTimeout deadline. diff --git a/packages/vm-agent/internal/bootstrap/bootstrap_test.go b/packages/vm-agent/internal/bootstrap/bootstrap_test.go index ddbad04fd6..fa5c4c4fc7 100644 --- a/packages/vm-agent/internal/bootstrap/bootstrap_test.go +++ b/packages/vm-agent/internal/bootstrap/bootstrap_test.go @@ -2108,6 +2108,99 @@ func TestPrepareWorkspaceReturnsFallbackFlag(t *testing.T) { } } +func TestPrepareWorkspaceLightweightSkipsRepoDevcontainerBuild(t *testing.T) { + mockBinDir := t.TempDir() + callLog := filepath.Join(t.TempDir(), "devcontainer-calls.log") + mockDevcontainer := filepath.Join(mockBinDir, "devcontainer") + mockScript := fmt.Sprintf(`#!/bin/sh +echo "$@" >> %s +if [ "$1" = "up" ]; then + for arg in "$@"; do + if [ "$arg" = "--override-config" ]; then + exit 0 + fi + done + echo "repo build should have been skipped" >&2 + exit 1 +fi +exit 0 +`, callLog) + if err := os.WriteFile(mockDevcontainer, []byte(mockScript), 0o755); err != nil { + t.Fatalf("failed to write mock devcontainer command: %v", err) + } + t.Setenv("PATH", mockBinDir+":"+os.Getenv("PATH")) + + workspaceID := "ws-prepare-lightweight" + workspaceDir := t.TempDir() + devcontainerDir := filepath.Join(workspaceDir, ".devcontainer") + if err := os.MkdirAll(devcontainerDir, 0o755); err != nil { + t.Fatalf("failed to create devcontainer dir: %v", err) + } + if err := os.WriteFile(filepath.Join(devcontainerDir, "devcontainer.json"), []byte(`{"image":"node:20"}`), 0o644); err != nil { + t.Fatalf("failed to write devcontainer config: %v", err) + } + + readyWorkspaceProfile := "" + controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/ready") { + var payload struct { + WorkspaceProfile string `json:"workspaceProfile"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("failed to decode ready payload: %v", err) + } + readyWorkspaceProfile = payload.WorkspaceProfile + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer controlPlane.Close() + + cfg := &config.Config{ + ControlPlaneURL: controlPlane.URL, + WorkspaceID: workspaceID, + CallbackToken: "cb-token", + WorkspaceDir: workspaceDir, + ContainerMode: false, + DefaultDevcontainerConfigPath: filepath.Join(t.TempDir(), "default-devcontainer.json"), + DefaultDevcontainerImage: "mcr.microsoft.com/devcontainers/base:ubuntu", + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + recoveryMode, err := PrepareWorkspace(ctx, cfg, ProvisionState{Lightweight: true}, nil) + if err != nil { + t.Fatalf("PrepareWorkspace returned error: %v", err) + } + if recoveryMode { + t.Fatal("expected lightweight startup to report recoveryMode=false") + } + if readyWorkspaceProfile != "lightweight" { + t.Fatalf("expected ready payload workspaceProfile=lightweight, got %q", readyWorkspaceProfile) + } + + calls, err := os.ReadFile(callLog) + if err != nil { + t.Fatalf("failed to read devcontainer calls: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(calls)), "\n") + upCalls := 0 + for _, line := range lines { + if strings.HasPrefix(line, "up ") { + upCalls++ + if !strings.Contains(line, "--override-config") { + t.Fatalf("expected lightweight devcontainer up to use default override config, got %q", line) + } + } + } + if upCalls != 1 { + t.Fatalf("expected exactly one lightweight devcontainer up call, got %d calls:\n%s", upCalls, string(calls)) + } +} + func TestEnsureDevcontainerReadyFallsBackOnRepoConfigFailure(t *testing.T) { // Mock devcontainer CLI that fails on first call (repo config) // but succeeds on second call (default config). diff --git a/packages/vm-agent/internal/bootstrap/pipeline.go b/packages/vm-agent/internal/bootstrap/pipeline.go new file mode 100644 index 0000000000..e65165dbc0 --- /dev/null +++ b/packages/vm-agent/internal/bootstrap/pipeline.go @@ -0,0 +1,567 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + + "github.com/workspace/vm-agent/internal/bootlog" + "github.com/workspace/vm-agent/internal/config" +) + +type StepStatus string + +const ( + StepCompleted StepStatus = "completed" + StepSkipped StepStatus = "skipped" + StepWarning StepStatus = "warning" +) + +type StepResult struct { + Status StepStatus + Message string + Err error +} + +type bootstrapStep struct { + name string + required bool + startMessage string + successMessage string + failureMessage string + run func(context.Context, *workspaceBootstrapContext) StepResult +} + +type workspaceBootstrapContext struct { + cfg *config.Config + reporter *bootlog.Reporter + provision ProvisionState + bootstrap *bootstrapState + + volumeName string + credentialHelperHostPath string + repoHasDevcontainerConfig bool + effectiveWorkspaceProfile string + devcontainerCacheRef string + usedFallback bool + recoveryMode bool + readyStatus string + + cleanup cleanupStack +} + +type cleanupAction struct { + name string + run func() +} + +type cleanupStack struct { + actions []cleanupAction + disarmed bool +} + +func (s *cleanupStack) register(name string, run func()) { + if run == nil { + return + } + s.actions = append(s.actions, cleanupAction{name: name, run: run}) +} + +func (s *cleanupStack) disarm() { + s.disarmed = true +} + +func (s *cleanupStack) runFailureCleanup() { + if s.disarmed { + return + } + for i := len(s.actions) - 1; i >= 0; i-- { + action := s.actions[i] + func() { + defer func() { + if recovered := recover(); recovered != nil { + slog.Warn("Bootstrap cleanup panicked", "name", action.name, "panic", recovered) + } + }() + action.run() + }() + } +} + +func runBootstrapPlan(ctx context.Context, plan []bootstrapStep, bootstrapCtx *workspaceBootstrapContext) error { + for _, step := range plan { + result := runBootstrapStep(ctx, step, bootstrapCtx) + if result.Err == nil || !step.required { + continue + } + bootstrapCtx.cleanup.runFailureCleanup() + return result.Err + } + return nil +} + +func runBootstrapStep(ctx context.Context, step bootstrapStep, bootstrapCtx *workspaceBootstrapContext) StepResult { + if step.run == nil { + return StepResult{Status: StepSkipped} + } + + if step.name != "" && step.startMessage != "" { + bootstrapCtx.reporter.Log(step.name, "started", step.startMessage) + } + + result := step.run(ctx, bootstrapCtx) + if result.Status == "" { + result.Status = StepCompleted + } + + if result.Err != nil { + message := result.Message + if message == "" { + message = step.failureMessage + } + if message == "" { + message = result.Err.Error() + } + if step.name != "" { + bootstrapCtx.reporter.Log(step.name, "failed", message, result.Err.Error()) + } + return result + } + + if result.Status == StepWarning { + message := result.Message + if message == "" { + message = step.failureMessage + } + if message != "" && step.name != "" { + bootstrapCtx.reporter.Log(step.name, "failed", message) + } + return result + } + + if result.Status == StepSkipped { + return result + } + + message := result.Message + if message == "" { + message = step.successMessage + } + if step.name != "" && message != "" { + bootstrapCtx.reporter.Log(step.name, "completed", message) + } + return result +} + +func okResult() StepResult { + return StepResult{Status: StepCompleted} +} + +func errResult(err error) StepResult { + return StepResult{Status: StepCompleted, Err: err} +} + +func warningResult(message string, err error) StepResult { + return StepResult{Status: StepWarning, Message: message, Err: err} +} + +func bootstrapTokenPlan() []bootstrapStep { + return []bootstrapStep{ + loadOrRedeemBootstrapStateStep(), + validateCallbackTokenStep(), + ensureWorkspaceVolumeStep(), + ensureRepositoryStep(), + writeCredentialHelperStep(), + startBootstrapDevcontainerStep(), + injectAptConfigStep(), + ensureGitHubCLIStep(), + ensureGitCredentialHelperStep(), + ensureGitIdentityStep(), + ensureSAMEnvironmentStep(), + resolveBootstrapReadyStatusStep(), + disarmCredentialCleanupStep(), + markWorkspaceReadyStep(), + } +} + +func prepareWorkspacePlan() []bootstrapStep { + return []bootstrapStep{ + useProvisionStateStep(), + ensureWorkspaceVolumeStep(), + ensureRepositoryStep(), + detectRepositoryDevcontainerStep(), + resolveEffectiveWorkspaceProfileStep(), + writeCredentialHelperStep(), + prepareDevcontainerCacheStep(), + startPrepareWorkspaceDevcontainerStep(), + injectAptConfigStep(), + ensureGitHubCLIStep(), + ensureGitCredentialHelperStep(), + ensureGitIdentityStep(), + ensureSAMEnvironmentStep(), + ensureProjectRuntimeAssetsStep(), + disarmCredentialCleanupStep(), + markWorkspaceReadyStep(), + } +} + +func loadOrRedeemBootstrapStateStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + state, err := loadState(b.cfg.BootstrapStatePath) + if err != nil { + return errResult(fmt.Errorf("failed to load bootstrap state: %w", err)) + } + if state != nil { + if state.WorkspaceID != b.cfg.WorkspaceID { + return errResult(fmt.Errorf("bootstrap state workspace mismatch: expected %s, found %s", b.cfg.WorkspaceID, state.WorkspaceID)) + } + slog.Info("Using cached bootstrap state", "path", b.cfg.BootstrapStatePath) + b.bootstrap = state + b.cfg.CallbackToken = state.CallbackToken + b.reporter.SetToken(state.CallbackToken) + return okResult() + } + + b.reporter.Log("bootstrap_redeem", "started", "Redeeming bootstrap credentials") + state, err = redeemBootstrapTokenWithRetry(ctx, b.cfg) + if err != nil { + return errResult(err) + } + b.bootstrap = state + b.cfg.CallbackToken = state.CallbackToken + b.reporter.SetToken(state.CallbackToken) + b.reporter.Log("bootstrap_redeem", "completed", "Bootstrap credentials redeemed") + if err := saveState(b.cfg.BootstrapStatePath, state); err != nil { + return errResult(fmt.Errorf("failed to persist bootstrap state: %w", err)) + } + return okResult() + }, + } +} + +func useProvisionStateStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + b.bootstrap = &bootstrapState{ + WorkspaceID: b.cfg.WorkspaceID, + CallbackToken: b.cfg.CallbackToken, + GitHubToken: strings.TrimSpace(b.provision.GitHubToken), + GitUserName: strings.TrimSpace(b.provision.GitUserName), + GitUserEmail: strings.TrimSpace(b.provision.GitUserEmail), + GitHubID: strings.TrimSpace(b.provision.GitHubID), + } + return okResult() + }, + } +} + +func validateCallbackTokenStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + if b.cfg.CallbackToken == "" { + return errResult(errors.New("callback token is missing after bootstrap")) + } + return okResult() + }, + } +} + +func ensureWorkspaceVolumeStep() bootstrapStep { + return bootstrapStep{ + name: "volume_create", + required: true, + startMessage: "Creating workspace volume", + successMessage: "Workspace volume ready", + failureMessage: "Volume creation failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + if !b.cfg.ContainerMode { + return StepResult{Status: StepSkipped} + } + volumeName, err := ensureVolumeReady(ctx, b.cfg.WorkspaceID) + if err != nil { + return errResult(err) + } + b.volumeName = volumeName + return okResult() + }, + } +} + +func ensureRepositoryStep() bootstrapStep { + return bootstrapStep{ + name: "git_clone", + required: true, + startMessage: "Cloning repository", + successMessage: "Repository cloned", + failureMessage: "Repository clone failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + return errResult(ensureRepositoryReady(ctx, b.cfg, b.bootstrap, b.volumeName)) + }, + } +} + +func detectRepositoryDevcontainerStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + b.repoHasDevcontainerConfig = hasDevcontainerConfig(b.cfg.WorkspaceDir) + return okResult() + }, + } +} + +func resolveEffectiveWorkspaceProfileStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + if b.provision.Lightweight || (b.provision.DevcontainerConfigName == "" && !b.repoHasDevcontainerConfig) { + b.effectiveWorkspaceProfile = "lightweight" + } + return okResult() + }, + } +} + +func writeCredentialHelperStep() bootstrapStep { + return bootstrapStep{ + name: "git_credential_helper", + required: false, + failureMessage: "Credential helper setup failed — git auth may be unavailable in lifecycle hooks", + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + credHelperHostPath, err := writeCredentialHelperToHost(b.cfg) + if err != nil { + slog.Warn("Failed to write credential helper to host (non-fatal)", "error", err) + return warningResult("Credential helper setup failed — git auth may be unavailable in lifecycle hooks", err) + } + b.credentialHelperHostPath = credHelperHostPath + if credHelperHostPath != "" { + b.cleanup.register("credential helper", func() { + RemoveCredentialHelperFromHost(b.cfg.WorkspaceID) + }) + } + return StepResult{Status: StepSkipped} + }, + } +} + +func prepareDevcontainerCacheStep() bootstrapStep { + return bootstrapStep{ + required: false, + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + if !b.cfg.DevcontainerCacheEnabled || b.provision.Lightweight || !b.repoHasDevcontainerConfig { + return StepResult{Status: StepSkipped} + } + cacheRef, err := prepareDevcontainerCache(ctx, b.cfg, b.bootstrap.GitHubToken, b.provision.DevcontainerConfigName) + if err != nil { + slog.Warn("Cache registry login failed (caching disabled for this build)", "registry", b.cfg.DevcontainerCacheRegistry, "error", err) + return StepResult{Status: StepWarning, Message: "Devcontainer cache unavailable"} + } + b.devcontainerCacheRef = cacheRef + if cacheRef != "" { + b.reporter.Log("devcontainer_cache", "started", "Checking devcontainer cache") + } + return okResult() + }, + } +} + +func startBootstrapDevcontainerStep() bootstrapStep { + return bootstrapStep{ + name: "devcontainer_up", + required: true, + startMessage: "Building devcontainer", + failureMessage: "Devcontainer build failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + b.reporter.Log("devcontainer_wait", "started", "Waiting for devcontainer CLI") + usedFallback, err := ensureDevcontainerReady(ctx, b.cfg, b.volumeName, b.credentialHelperHostPath, "", "") + if err != nil { + return errResult(err) + } + b.usedFallback = usedFallback + if usedFallback { + return StepResult{Status: StepCompleted, Message: "Devcontainer ready (fallback to default image)"} + } + return StepResult{Status: StepCompleted, Message: "Devcontainer ready"} + }, + } +} + +func startPrepareWorkspaceDevcontainerStep() bootstrapStep { + return bootstrapStep{ + name: "devcontainer_up", + required: true, + failureMessage: "Devcontainer build failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + if b.provision.Lightweight { + b.reporter.Log("devcontainer_up", "started", "Starting lightweight container (skipping devcontainer build)") + slog.Info("Lightweight mode: forcing fallback image, skipping devcontainer build", "workspaceID", b.cfg.WorkspaceID) + usedFallback, err := ensureDevcontainerFallback(ctx, b.cfg, b.volumeName, b.credentialHelperHostPath) + if err != nil { + return StepResult{Err: err, Message: "Lightweight container startup failed"} + } + b.usedFallback = usedFallback + return StepResult{Status: StepCompleted, Message: "Lightweight container ready"} + } + + b.reporter.Log("devcontainer_up", "started", "Building devcontainer") + usedFallback, err := ensureDevcontainerReady(ctx, b.cfg, b.volumeName, b.credentialHelperHostPath, b.provision.DevcontainerConfigName, b.devcontainerCacheRef) + if err != nil { + return errResult(err) + } + b.usedFallback = usedFallback + b.recoveryMode = usedFallback + if markerFound, markerErr := hasBuildErrorMarker(b.cfg); markerErr != nil { + slog.Warn("Failed to inspect build error marker", "workspaceID", b.cfg.WorkspaceID, "error", markerErr) + } else if markerFound { + b.recoveryMode = true + } + if usedFallback { + return StepResult{Status: StepCompleted, Message: "Devcontainer ready (fallback to default image)"} + } + return StepResult{Status: StepCompleted, Message: "Devcontainer ready"} + }, + } +} + +func injectAptConfigStep() bootstrapStep { + return bootstrapStep{ + required: false, + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + containerID, findErr := findDevcontainerID(ctx, b.cfg) + if findErr != nil { + slog.Debug("Could not find devcontainer for apt config injection (non-fatal)", "error", findErr) + return StepResult{Status: StepSkipped} + } + injectAptRetryConfig(ctx, containerID) + injectAptMirrorConfig(ctx, b.cfg, containerID) + return okResult() + }, + } +} + +func ensureGitHubCLIStep() bootstrapStep { + return bootstrapStep{ + name: "gh_cli", + required: false, + startMessage: "Checking GitHub CLI availability", + successMessage: "GitHub CLI available", + failureMessage: "GitHub CLI install failed (non-fatal)", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + if err := ensureGitHubCLI(ctx, b.cfg); err != nil { + slog.Warn("GitHub CLI install failed (non-fatal)", "error", err) + return warningResult("GitHub CLI install failed (non-fatal)", err) + } + return okResult() + }, + } +} + +func ensureGitCredentialHelperStep() bootstrapStep { + return bootstrapStep{ + name: "git_creds", + required: true, + startMessage: "Configuring git credentials", + successMessage: "Git credentials configured", + failureMessage: "Git credential setup failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + return errResult(ensureGitCredentialHelper(ctx, b.cfg)) + }, + } +} + +func ensureGitIdentityStep() bootstrapStep { + return bootstrapStep{ + name: "git_identity", + required: true, + startMessage: "Configuring git identity", + successMessage: "Git identity configured", + failureMessage: "Git identity setup failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + return errResult(ensureGitIdentity(ctx, b.cfg, b.bootstrap)) + }, + } +} + +func ensureSAMEnvironmentStep() bootstrapStep { + return bootstrapStep{ + name: "sam_env", + required: false, + startMessage: "Configuring SAM environment", + successMessage: "SAM environment configured", + failureMessage: "SAM environment setup failed", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + if err := ensureSAMEnvironment(ctx, b.cfg, b.bootstrap.GitHubToken); err != nil { + slog.Warn("SAM environment setup failed (non-fatal)", "error", err) + return warningResult("SAM environment setup failed", err) + } + return okResult() + }, + } +} + +func ensureProjectRuntimeAssetsStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + return errResult(ensureProjectRuntimeAssets(ctx, b.cfg, b.provision.ProjectEnvVars, b.provision.ProjectFiles)) + }, + } +} + +func resolveBootstrapReadyStatusStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + readyStatus := workspaceReadyStatusRunning + if recovery, recoveryErr := hasBuildErrorMarker(b.cfg); recoveryErr != nil { + slog.Warn("Failed to inspect build error marker", "workspaceID", b.cfg.WorkspaceID, "error", recoveryErr) + } else if recovery { + readyStatus = workspaceReadyStatusRecovery + } + if b.usedFallback { + readyStatus = workspaceReadyStatusRecovery + } + b.readyStatus = readyStatus + return okResult() + }, + } +} + +func disarmCredentialCleanupStep() bootstrapStep { + return bootstrapStep{ + required: true, + run: func(_ context.Context, b *workspaceBootstrapContext) StepResult { + b.cleanup.disarm() + return okResult() + }, + } +} + +func markWorkspaceReadyStep() bootstrapStep { + return bootstrapStep{ + name: "workspace_ready", + required: true, + startMessage: "Marking workspace ready", + successMessage: "Workspace is ready", + failureMessage: "Failed to mark workspace ready", + run: func(ctx context.Context, b *workspaceBootstrapContext) StepResult { + readyStatus := b.readyStatus + if readyStatus == "" { + readyStatus = workspaceReadyStatusRunning + if b.recoveryMode { + readyStatus = workspaceReadyStatusRecovery + } + } + if err := markWorkspaceReady(ctx, b.cfg, readyStatus, b.effectiveWorkspaceProfile); err != nil { + return errResult(&CallbackError{Err: err, Status: readyStatus}) + } + return okResult() + }, + } +} diff --git a/packages/vm-agent/internal/bootstrap/pipeline_test.go b/packages/vm-agent/internal/bootstrap/pipeline_test.go new file mode 100644 index 0000000000..f964883e7e --- /dev/null +++ b/packages/vm-agent/internal/bootstrap/pipeline_test.go @@ -0,0 +1,260 @@ +package bootstrap + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/workspace/vm-agent/internal/bootlog" + "github.com/workspace/vm-agent/internal/config" +) + +type recordedBootlogEvent struct { + step string + status string + message string +} + +type recordingBroadcaster struct { + events []recordedBootlogEvent +} + +func (b *recordingBroadcaster) Broadcast(step, status, message string, _ ...string) { + b.events = append(b.events, recordedBootlogEvent{step: step, status: status, message: message}) +} + +func TestRunBootstrapPlanExecutesStepsInOrder(t *testing.T) { + var calls []string + plan := []bootstrapStep{ + testStep("first", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "first") + return okResult() + }), + testStep("second", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "second") + return okResult() + }), + testStep("third", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "third") + return okResult() + }), + } + + if err := runBootstrapPlan(context.Background(), plan, &workspaceBootstrapContext{}); err != nil { + t.Fatalf("runBootstrapPlan returned error: %v", err) + } + + want := []string{"first", "second", "third"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("unexpected call order: got %v, want %v", calls, want) + } +} + +func TestRunBootstrapPlanStopsOnRequiredStepFailure(t *testing.T) { + stepErr := errors.New("required failed") + var calls []string + plan := []bootstrapStep{ + testStep("first", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "first") + return okResult() + }), + testStep("second", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "second") + return errResult(stepErr) + }), + testStep("third", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "third") + return okResult() + }), + } + + err := runBootstrapPlan(context.Background(), plan, &workspaceBootstrapContext{}) + if !errors.Is(err, stepErr) { + t.Fatalf("expected required step error, got %v", err) + } + + want := []string{"first", "second"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("unexpected call order after failure: got %v, want %v", calls, want) + } +} + +func TestRunBootstrapPlanContinuesAfterOptionalStepFailure(t *testing.T) { + var calls []string + plan := []bootstrapStep{ + testStep("first", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "first") + return okResult() + }), + testStep("second", false, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "second") + return warningResult("non-fatal failed", errors.New("optional failed")) + }), + testStep("third", true, func(context.Context, *workspaceBootstrapContext) StepResult { + calls = append(calls, "third") + return okResult() + }), + } + + if err := runBootstrapPlan(context.Background(), plan, &workspaceBootstrapContext{}); err != nil { + t.Fatalf("runBootstrapPlan returned error: %v", err) + } + + want := []string{"first", "second", "third"} + if !reflect.DeepEqual(calls, want) { + t.Fatalf("unexpected call order: got %v, want %v", calls, want) + } +} + +func TestRunBootstrapPlanRunsCleanupOnFailureOnly(t *testing.T) { + var cleanupCalls []string + failPlan := []bootstrapStep{ + testStep("register", true, func(_ context.Context, b *workspaceBootstrapContext) StepResult { + b.cleanup.register("first", func() { cleanupCalls = append(cleanupCalls, "first") }) + b.cleanup.register("second", func() { cleanupCalls = append(cleanupCalls, "second") }) + return okResult() + }), + testStep("fail", true, func(context.Context, *workspaceBootstrapContext) StepResult { + return errResult(errors.New("boom")) + }), + } + + if err := runBootstrapPlan(context.Background(), failPlan, &workspaceBootstrapContext{}); err == nil { + t.Fatal("expected failure") + } + if want := []string{"second", "first"}; !reflect.DeepEqual(cleanupCalls, want) { + t.Fatalf("unexpected cleanup order: got %v, want %v", cleanupCalls, want) + } + + cleanupCalls = nil + successPlan := []bootstrapStep{ + testStep("register", true, func(_ context.Context, b *workspaceBootstrapContext) StepResult { + b.cleanup.register("first", func() { cleanupCalls = append(cleanupCalls, "first") }) + return okResult() + }), + } + if err := runBootstrapPlan(context.Background(), successPlan, &workspaceBootstrapContext{}); err != nil { + t.Fatalf("unexpected success plan error: %v", err) + } + if len(cleanupCalls) != 0 { + t.Fatalf("cleanup should not run on success, got %v", cleanupCalls) + } +} + +func TestRunBootstrapPlanDoesNotCleanupAfterDisarm(t *testing.T) { + var cleanupCalled bool + plan := []bootstrapStep{ + testStep("register", true, func(_ context.Context, b *workspaceBootstrapContext) StepResult { + b.cleanup.register("credential helper", func() { cleanupCalled = true }) + return okResult() + }), + disarmCredentialCleanupStep(), + testStep("callback", true, func(context.Context, *workspaceBootstrapContext) StepResult { + return errResult(&CallbackError{Err: errors.New("ready failed"), Status: workspaceReadyStatusRunning}) + }), + } + + if err := runBootstrapPlan(context.Background(), plan, &workspaceBootstrapContext{}); err == nil { + t.Fatal("expected callback failure") + } + if cleanupCalled { + t.Fatal("cleanup should stay disarmed after workspace provisioning succeeds") + } +} + +func TestRunBootstrapPlanReporterEventOrder(t *testing.T) { + broadcaster := &recordingBroadcaster{} + reporter := bootlog.New("http://example.invalid", "ws-pipeline") + reporter.SetBroadcaster(broadcaster) + + plan := []bootstrapStep{ + { + name: "required", + required: true, + startMessage: "starting required", + successMessage: "required complete", + run: func(context.Context, *workspaceBootstrapContext) StepResult { + return okResult() + }, + }, + { + name: "optional", + required: false, + startMessage: "starting optional", + failureMessage: "optional failed", + run: func(context.Context, *workspaceBootstrapContext) StepResult { + return warningResult("optional failed", errors.New("optional failed")) + }, + }, + { + name: "after", + required: true, + startMessage: "starting after", + successMessage: "after complete", + run: func(context.Context, *workspaceBootstrapContext) StepResult { + return okResult() + }, + }, + } + + if err := runBootstrapPlan(context.Background(), plan, &workspaceBootstrapContext{reporter: reporter}); err != nil { + t.Fatalf("runBootstrapPlan returned error: %v", err) + } + + got := broadcaster.events + want := []recordedBootlogEvent{ + {step: "required", status: "started", message: "starting required"}, + {step: "required", status: "completed", message: "required complete"}, + {step: "optional", status: "started", message: "starting optional"}, + {step: "optional", status: "failed", message: "optional failed"}, + {step: "after", status: "started", message: "starting after"}, + {step: "after", status: "completed", message: "after complete"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected reporter events:\ngot: %#v\nwant: %#v", got, want) + } +} + +func TestPrepareDevcontainerCacheStepIsNonFatal(t *testing.T) { + mockBinDir := t.TempDir() + mockDocker := filepath.Join(mockBinDir, "docker") + if err := os.WriteFile(mockDocker, []byte("#!/bin/sh\necho login failed >&2\nexit 1\n"), 0o755); err != nil { + t.Fatalf("failed to write mock docker command: %v", err) + } + t.Setenv("PATH", mockBinDir+":"+os.Getenv("PATH")) + + var afterCacheRan bool + plan := []bootstrapStep{ + prepareDevcontainerCacheStep(), + testStep("after-cache", true, func(context.Context, *workspaceBootstrapContext) StepResult { + afterCacheRan = true + return okResult() + }), + } + bootstrapCtx := &workspaceBootstrapContext{ + cfg: &config.Config{ + DevcontainerCacheEnabled: true, + DevcontainerCacheRegistry: "ghcr.io", + Repository: "owner/repo", + }, + bootstrap: &bootstrapState{GitHubToken: "token"}, + repoHasDevcontainerConfig: true, + } + + if err := runBootstrapPlan(context.Background(), plan, bootstrapCtx); err != nil { + t.Fatalf("cache failure should be non-fatal, got %v", err) + } + if !afterCacheRan { + t.Fatal("expected plan to continue after cache preparation failure") + } + if bootstrapCtx.devcontainerCacheRef != "" { + t.Fatalf("expected cache ref to be cleared after login failure, got %q", bootstrapCtx.devcontainerCacheRef) + } +} + +func testStep(name string, required bool, run func(context.Context, *workspaceBootstrapContext) StepResult) bootstrapStep { + return bootstrapStep{name: name, required: required, run: run} +} diff --git a/tasks/backlog/2026-06-18-vm-agent-bootstrap-pipeline.md b/tasks/archive/2026-06-18-vm-agent-bootstrap-pipeline.md similarity index 76% rename from tasks/backlog/2026-06-18-vm-agent-bootstrap-pipeline.md rename to tasks/archive/2026-06-18-vm-agent-bootstrap-pipeline.md index 2b0ae59c2c..e1b71bf664 100644 --- a/tasks/backlog/2026-06-18-vm-agent-bootstrap-pipeline.md +++ b/tasks/archive/2026-06-18-vm-agent-bootstrap-pipeline.md @@ -27,17 +27,17 @@ ## Implementation Checklist -- [ ] Characterize existing bootstrap behavior and test seams in `packages/vm-agent/internal/bootstrap`. -- [ ] Introduce a small bootstrap-local context, step result/status model, and cleanup stack. -- [ ] Add a step runner with explicit ordering, fatal/non-fatal semantics, cleanup-on-failure behavior, and reporter phase handling. -- [ ] Refactor `Run` and `PrepareWorkspace` to assemble explicit plans from shared steps while preserving behavior. -- [ ] Decompose `ensureDevcontainerReady` into focused helpers where practical. -- [ ] Add tests for step ordering and reporter event order. -- [ ] Add tests for required step failure, optional step warning/continuation, and cleanup behavior. -- [ ] Add tests for lightweight behavior, cache non-fatal behavior, named devcontainer config failure, and fallback behavior. -- [ ] Run focused bootstrap tests from `packages/vm-agent`. -- [ ] Run related/full VM agent package tests if the environment supports it. -- [ ] Create a draft/open PR marked `DO NOT MERGE / DO NOT DEPLOY TO STAGING`. +- [x] Characterize existing bootstrap behavior and test seams in `packages/vm-agent/internal/bootstrap`. +- [x] Introduce a small bootstrap-local context, step result/status model, and cleanup stack. +- [x] Add a step runner with explicit ordering, fatal/non-fatal semantics, cleanup-on-failure behavior, and reporter phase handling. +- [x] Refactor `Run` and `PrepareWorkspace` to assemble explicit plans from shared steps while preserving behavior. +- [x] Decompose `ensureDevcontainerReady` into focused helpers where practical. +- [x] Add tests for step ordering and reporter event order. +- [x] Add tests for required step failure, optional step warning/continuation, and cleanup behavior. +- [x] Add tests for lightweight behavior, cache non-fatal behavior, named devcontainer config failure, and fallback behavior. +- [x] Run focused bootstrap tests from `packages/vm-agent`. +- [x] Run related/full VM agent package tests if the environment supports it. +- [x] Create a draft/open PR marked `DO NOT MERGE / DO NOT DEPLOY TO STAGING`. ## Acceptance Criteria