From e8f94ec86b001c83112211f5ba16c48df9979f09 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Sat, 28 Mar 2026 16:01:24 +0100 Subject: [PATCH 01/14] Improve devenv shell bootstrap caching --- CHANGELOG.md | 4 + devenv.nix | 9 +- .../otel/dashboards/shell-entry.jsonnet | 18 +- nix/devenv-modules/tasks/shared/genie.nix | 57 +++++- nix/devenv-modules/tasks/shared/megarepo.nix | 67 +++++-- nix/devenv-modules/tasks/shared/pnpm.nix | 38 +++- nix/devenv-modules/tasks/shared/setup.nix | 180 ++++++++++++++++-- nix/devenv-modules/tasks/shared/ts.nix | 50 ++++- 8 files changed, 373 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3e6102ff..8ab710b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,10 @@ All notable changes to this project will be documented in this file. - Unblocks composed-root `pnpm-workspace.yaml` generation in downstream megarepos that import `@overeng/utils` - **@overeng/genie**: Use cwd-relative lock directory instead of shared `/tmp/genie-locks/` to fix `EACCES` errors in multi-user CI environments (#520) - **@overeng/tui-react**: Format timeline timestamps as human-readable durations (e.g. `6m 18s / 16m 21s`) instead of raw seconds (`377.9s / 980.6s`) in `TuiStoryPreview` (#472) +- **devenv/tasks**: make warm shell bootstrap commit-scoped and remove `ts:emit` from shell entry + - Adds an outer `setup:auto` cache so warm `devenv shell` skips unchanged bootstrap work instead of traversing `pnpm:install`, `genie:run`, and `mr:apply` on every entry + - Switches shell bootstrap from `mr:sync` to initial `mr:apply` so a fresh worktree is normalized without fetching on every shell + - Speeds up warm task status paths by using direct `mr status`, fingerprint-based `genie:run` caching, lighter `pnpm:install` status checks, and a `ts:emit` graph that excludes `noEmit` references at emit time - **@overeng/genie**: Validate GitHub Actions `runs-on` labels before emitting workflow YAML - Fails `genie` when workflow jobs serialize non-string, empty, or stale placeholder runner labels like `null` / `...=undefined` - Prevents CI helper API drift from silently generating invalid workflow files that only fail later in GitHub Actions diff --git a/devenv.nix b/devenv.nix index 600f690ca..92cf42b91 100644 --- a/devenv.nix +++ b/devenv.nix @@ -351,8 +351,12 @@ in optionalTasks = [ "pnpm:install" "genie:run" - "mr:fetch-apply" - "ts:emit" + "mr:apply" + ]; + innerCacheDirs = [ + "pnpm-install" + "genie-run" + "mr-apply" ]; completionsCliNames = [ "genie" @@ -401,6 +405,7 @@ in tasks."genie:check".after = [ "pnpm:install" ]; tasks."lint:check:genie".after = [ "pnpm:install" ]; tasks."mr:fetch-apply".after = [ "pnpm:install" ]; + tasks."mr:apply".after = [ "pnpm:install" ]; tasks."gh:apply-settings" = { after = [ "genie:run" ]; diff --git a/nix/devenv-modules/otel/dashboards/shell-entry.jsonnet b/nix/devenv-modules/otel/dashboards/shell-entry.jsonnet index 72ec7fbe6..392fa4adb 100644 --- a/nix/devenv-modules/otel/dashboards/shell-entry.jsonnet +++ b/nix/devenv-modules/otel/dashboards/shell-entry.jsonnet @@ -1,7 +1,7 @@ // Shell Entry (enterShell) dashboard // How long do shell entry tasks take, with breakdown by task. // -// Shell entry runs optional tasks: pnpm:install, genie:run, mr:fetch-apply, ts:emit +// Shell entry runs optional tasks: pnpm:install, genie:run, mr:apply // These tasks are only executed when their dependencies change (git hash caching). // Use FORCE_SETUP=1 to force re-run even when cached. local g = import 'g.libsonnet'; @@ -30,7 +30,7 @@ local traceTable(title, query, limit=50) = g.dashboard.new('Shell Entry Performance') + g.dashboard.withUid('otel-shell-entry') -+ g.dashboard.withDescription('Performance breakdown of devenv shell entry tasks (pnpm:install, genie:run, mr:fetch-apply, ts:emit)') ++ g.dashboard.withDescription('Performance breakdown of devenv shell entry tasks (pnpm:install, genie:run, mr:apply)') + g.dashboard.graphTooltip.withSharedCrosshair() + g.dashboard.withTimezone('browser') + g.dashboard.withPanels( @@ -39,8 +39,8 @@ g.dashboard.new('Shell Entry Performance') g.panel.row.new('Shell Entry Tasks'), traceTable( - 'All shell entry tasks (pnpm:install, genie:run, mr:fetch-apply, ts:emit)', - '{resource.service.name="dt-task" && name=~"pnpm:install|genie:run|mr:fetch-apply|ts:emit"}', + 'All shell entry tasks (pnpm:install, genie:run, mr:apply)', + '{resource.service.name="dt-task" && name=~"pnpm:install|genie:run|mr:apply"}', 50, ), @@ -60,14 +60,8 @@ g.dashboard.new('Shell Entry Performance') ), traceTable( - 'mr:fetch-apply', - '{resource.service.name="dt-task" && name="mr:fetch-apply"}', - 30, - ), - - traceTable( - 'ts:emit', - '{resource.service.name="dt-task" && name="ts:emit"}', + 'mr:apply', + '{resource.service.name="dt-task" && name="mr:apply"}', 30, ), ], panelWidth=24, panelHeight=10) diff --git a/nix/devenv-modules/tasks/shared/genie.nix b/nix/devenv-modules/tasks/shared/genie.nix index 26b63596c..508cbfe8f 100644 --- a/nix/devenv-modules/tasks/shared/genie.nix +++ b/nix/devenv-modules/tasks/shared/genie.nix @@ -19,6 +19,37 @@ let genieTaskEnv = lib.optionalAttrs (megarepoStoreEnv != "") { MEGAREPO_STORE = megarepoStoreEnv; }; + cacheRoot = ".direnv/task-cache/genie-run"; + stateFile = "${cacheRoot}/state.hash"; + computeGenieStateHash = '' + compute_genie_state_hash() { + { + ${pkgs.findutils}/bin/find . \ + -type f \ + -name '*.genie.ts' \ + -not -path './.git/*' \ + -not -path './.direnv/*' \ + -not -path './.devenv/*' \ + -not -path './node_modules/*' \ + -print + ${pkgs.ripgrep}/bin/rg -l \ + --glob '!tmp/**' \ + --glob '!.git/**' \ + --glob '!.direnv/**' \ + --glob '!.devenv/**' \ + --glob '!node_modules/**' \ + '^// Source: .*\.genie\.ts|^# Source: .*\.genie\.ts' . + } \ + | LC_ALL=C sort -u \ + | while IFS= read -r file; do + [ -f "$file" ] || continue + printf '%s\n' "$file" + ${pkgs.coreutils}/bin/sha256sum "$file" | awk '{print $1}' + done \ + | ${pkgs.coreutils}/bin/sha256sum \ + | awk '{print $1}' + } + ''; tasks = { "genie:prepare" = { @@ -31,12 +62,30 @@ let description = "Generate config files from .genie.ts sources"; after = [ "genie:prepare" ]; env = genieTaskEnv; - exec = trace.exec "genie:run" "genie"; + exec = trace.exec "genie:run" '' + set -euo pipefail + mkdir -p ${lib.escapeShellArg cacheRoot} + ${computeGenieStateHash} + genie + cache_value="$(compute_genie_state_hash)" + tmp_file="$(mktemp)" + printf "%s" "$cache_value" > "$tmp_file" + if [ -f ${lib.escapeShellArg stateFile} ] && cmp -s "$tmp_file" ${lib.escapeShellArg stateFile}; then + rm "$tmp_file" + else + mv "$tmp_file" ${lib.escapeShellArg stateFile} + fi + ''; status = trace.status "genie:run" "binary" '' set -euo pipefail - # Skip when generated files are already up to date. - # Silence output to keep shell entry clean. - genie --check >/dev/null 2>&1 + if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + exit 0 + fi + [ -f ${lib.escapeShellArg stateFile} ] || exit 1 + ${computeGenieStateHash} + current_hash="$(compute_genie_state_hash)" + stored_hash="$(cat ${lib.escapeShellArg stateFile})" + [ "$current_hash" = "$stored_hash" ] ''; }; "genie:watch" = { diff --git a/nix/devenv-modules/tasks/shared/megarepo.nix b/nix/devenv-modules/tasks/shared/megarepo.nix index 3c2b2290c..6ee53d232 100644 --- a/nix/devenv-modules/tasks/shared/megarepo.nix +++ b/nix/devenv-modules/tasks/shared/megarepo.nix @@ -33,6 +33,26 @@ let bootstrapOnlyArgs = lib.concatMapStringsSep " " ( member: "--only ${lib.escapeShellArg member}" ) bootstrapMembers; + cacheRoot = ".direnv/task-cache/mr-apply"; + membersFile = "${cacheRoot}/members.txt"; + recordWorkspaceMembers = '' + set -o pipefail + mkdir -p ${lib.escapeShellArg cacheRoot} + tmp_members_file="$(mktemp)" + # Rewrite the manifest atomically so a failed `mr ls` never leaves behind + # an empty file that would make the warm-path output proof vacuous. + mr ls --output json \ + | ${jq} -r 'select(._tag == "Success") | .value.members[].name' \ + | while IFS= read -r member; do + [ -n "$member" ] || continue + case ",''${MEGAREPO_SKIP_MEMBERS:-}," in + *,"$member",*) continue ;; + esac + printf '%s\n' "$member" + done \ + | LC_ALL=C sort -u > "$tmp_members_file" + mv "$tmp_members_file" ${lib.escapeShellArg membersFile} + ''; # Single-pass jq script that compares megarepo.lock member commits against # a Nix lock file (devenv.lock or flake.lock). Handles multiple inputs @@ -122,6 +142,33 @@ let | .[].name ''; + mrStatusCheck = '' + # Use the already-installed source CLI here. `nix run ...#megarepo` adds a + # second eval/build hop to every warm status check. + if [ ! -f ./megarepo.kdl ] && [ ! -f ./megarepo.json ]; then + exit 0 + fi + + if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + [ -d ./repos ] || exit 1 + [ -f ${lib.escapeShellArg membersFile} ] || exit 1 + while IFS= read -r member; do + [ -n "$member" ] || continue + if [ ! -L "./repos/$member" ] && [ ! -d "./repos/$member" ]; then + exit 1 + fi + done < ${lib.escapeShellArg membersFile} + exit 0 + fi + + if [ ! -d ./repos ]; then + exit 1 + fi + + status_json=$(mr status --output json 2>/dev/null) || exit 1 + echo "$status_json" | ${jq} -e '(.workspaceSyncNeeded // false) == false' >/dev/null 2>&1 + ''; + tasks = { "mr:bootstrap" = { guard = "mr"; @@ -171,23 +218,9 @@ let ${loadCheckSkipMembersScript} build_mr_skip_args mr fetch --apply${if syncAll then " --all" else ""} "''${MR_SKIP_ARGS[@]}" + ${recordWorkspaceMembers} ''; - # Status: use `mr status --output json` to detect if workspace reconciliation is needed. - status = trace.status "mr:fetch-apply" "binary" '' - if [ ! -f ./megarepo.kdl ] && [ ! -f ./megarepo.json ]; then - exit 0 - fi - - # Fast check: if repos/ doesn't exist, definitely need sync - if [ ! -d ./repos ]; then - exit 1 - fi - - # Use mr status to check whether workspace needs mr apply - status_json=$(mr status --output json 2>/dev/null) || exit 1 - - echo "$status_json" | ${jq} -e '(.applyNeeded // false) == false' >/dev/null 2>&1 - ''; + status = trace.status "mr:fetch-apply" "binary" mrStatusCheck; }; "mr:lock" = { @@ -215,7 +248,9 @@ let ${loadCheckSkipMembersScript} build_mr_skip_args mr apply${if syncAll then " --all" else ""} "''${MR_SKIP_ARGS[@]}" + ${recordWorkspaceMembers} ''; + status = trace.status "mr:apply" "binary" mrStatusCheck; }; "mr:check" = { diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 5ba2bb3eb..34816e1f6 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -189,6 +189,26 @@ let } | compute_hash } ''; + computeProjectionStateHashFn = '' + compute_projection_state_hash() { + { + for node_modules_dir in node_modules ${nodeModulesPaths}; do + if [ -d "$node_modules_dir" ]; then + printf 'dir %s\n' "$node_modules_dir" + else + printf 'missing %s\n' "$node_modules_dir" + fi + done + + if [ -f node_modules/.modules.yaml ]; then + printf 'modules-yaml ' + sha256sum node_modules/.modules.yaml | awk '{print $1}' + else + printf 'modules-yaml missing\n' + fi + } | compute_hash + } + ''; runPnpmInstallFn = '' run_pnpm_install() { @@ -284,6 +304,7 @@ let # manifests. The fingerprint also includes the active GVS projection # root because pnpm 11 bakes absolute paths into `links/`. hash_file="${cacheRoot}/install-state.hash" + projection_hash_file="${cacheRoot}/projection-state.hash" lockfile="${cacheRoot}/pnpm-install.lock" exec 200>"$lockfile" @@ -315,6 +336,7 @@ let ${computeWorkspaceStateHash} ${computeInstallStateHashFn} + ${computeProjectionStateHashFn} ${preInstall} ${runPnpmInstallFn} @@ -391,27 +413,37 @@ let cache_value="$(compute_install_state_hash)" ${cache.writeCacheFile ''"$hash_file"''} + + cache_value="$(compute_projection_state_hash)" + ${cache.writeCacheFile ''"$projection_hash_file"''} ''; status = trace.status installTaskName "hash" '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} + if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + exit 0 + fi ${loadPnpmTaskHelpersFn} ${ensureLocalPnpmHomeFn} ${ensureLocalPnpmStoreDirFn} hash_file="${cacheRoot}/install-state.hash" + projection_hash_file="${cacheRoot}/projection-state.hash" - if [ ! -d node_modules ] || [ ! -f pnpm-lock.yaml ] || [ ! -f "$hash_file" ]; then + if [ ! -d node_modules ] || [ ! -f pnpm-lock.yaml ] || [ ! -f "$hash_file" ] || [ ! -f "$projection_hash_file" ] || [ ! -f node_modules/.modules.yaml ]; then exit 1 fi ${computeWorkspaceStateHash} ${computeInstallStateHashFn} + ${computeProjectionStateHashFn} current_hash="$(compute_install_state_hash)" + current_projection_hash="$(compute_projection_state_hash)" stored_hash="$(cat "$hash_file")" - if ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionHealthScript} ${healthCheckNodeModulesPaths}; then + stored_projection_hash="$(cat "$projection_hash_file")" + if [ "$current_hash" != "$stored_hash" ]; then exit 1 fi - if [ "$current_hash" != "$stored_hash" ]; then + if [ "$current_projection_hash" != "$stored_projection_hash" ]; then exit 1 fi exit 0 diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index b65059e3a..7e0a68837 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -30,6 +30,7 @@ requiredTasks ? [ ], optionalTasks ? [ ], completionsCliNames ? [ ], + innerCacheDirs ? [ ], skipDuringRebase ? true, }: { @@ -40,12 +41,16 @@ }: let cliGuard = import ../lib/cli-guard.nix { inherit pkgs; }; + cache = import ../lib/cache.nix { inherit config; }; git = "${pkgs.git}/bin/git"; userRequiredTasks = requiredTasks; userOptionalTasks = optionalTasks; completionsEnabled = completionsCliNames != [ ]; completionsTaskName = "setup:completions"; + setupRecordCacheTaskName = "setup:record-cache"; completionsCliList = lib.concatStringsSep " " completionsCliNames; + setupFingerprintFile = cache.mkCachePath "setup-fingerprint"; + setupGitHashFile = cache.mkCachePath "setup-git-hash"; completionsExec = '' shell="" if [ -n "''${FISH_VERSION:-}" ]; then @@ -96,6 +101,10 @@ let exit 0 ''; completionsStatus = '' + if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + exit 0 + fi + shell="" if [ -n "''${FISH_VERSION:-}" ]; then shell="fish" @@ -145,6 +154,125 @@ let setupOptionalTasks = userOptionalTasks ++ lib.optionals completionsEnabled [ completionsTaskName ]; setupTasks = setupRequiredTasks ++ setupOptionalTasks; allSetupTasks = setupTasks; + setupInnerCacheDirList = lib.concatMapStringsSep " " lib.escapeShellArg innerCacheDirs; + setupFingerprintEnv = '' + compute_setup_fingerprint() { + _setup_head=$(${git} rev-parse HEAD 2>/dev/null || echo "no-git") + _setup_generated_from_head=$( + ${git} grep -l -E '^// Source: .*\.genie\.ts|^# Source: .*\.genie\.ts' HEAD -- . 2>/dev/null || true + ) + _setup_dirty_files=$( + { + ${git} -c core.quotepath=off ls-files \ + --modified \ + --others \ + --exclude-standard \ + --deduplicate \ + -- \ + ':(glob)**/*.genie.ts' \ + ':(glob)**/package.json' 2>/dev/null || true + + for _setup_file in package.json pnpm-workspace.yaml pnpm-lock.yaml .npmrc megarepo.kdl megarepo.json megarepo.lock; do + if [ -f "$_setup_file" ] && ! ${git} ls-files --error-unmatch -- "$_setup_file" >/dev/null 2>&1; then + printf '%s\n' "$_setup_file" + elif ! ${git} diff --quiet -- "$_setup_file" 2>/dev/null; then + printf '%s\n' "$_setup_file" + fi + done + + printf '%s\n' "$_setup_generated_from_head" \ + | while IFS= read -r _setup_file; do + [ -n "$_setup_file" ] || continue + if [ ! -e "$_setup_file" ] || ! ${git} diff --quiet -- "$_setup_file" 2>/dev/null; then + printf '%s\n' "$_setup_file" + fi + done + } | LC_ALL=C sort -u + ) + + { + printf 'head %s\n' "$_setup_head" + + for _setup_file in package.json pnpm-workspace.yaml pnpm-lock.yaml .npmrc megarepo.kdl megarepo.json megarepo.lock; do + ${git} ls-files -s -- "$_setup_file" 2>/dev/null || true + done + + ${git} -c core.quotepath=off ls-files -s -- ':(glob)**/*.genie.ts' ':(glob)**/package.json' 2>/dev/null || true + + printf '%s\n' "$_setup_generated_from_head" \ + | while IFS= read -r _setup_file; do + [ -n "$_setup_file" ] || continue + ${git} ls-files -s -- "$_setup_file" 2>/dev/null || true + done + + printf '%s\n' "$_setup_dirty_files" \ + | while IFS= read -r _setup_file; do + [ -n "$_setup_file" ] || continue + if [ -f "$_setup_file" ]; then + printf 'dirty %s\n' "$_setup_file" + ${pkgs.coreutils}/bin/sha256sum "$_setup_file" | awk '{print $1}' + else + printf 'missing %s\n' "$_setup_file" + fi + done + } \ + | LC_ALL=C sort -u \ + | ${pkgs.coreutils}/bin/sha256sum \ + | awk '{print $1}' + } + + setup_outer_cache_hit() { + _setup_current_fingerprint="$1" + + if [ "''${FORCE_SETUP:-}" = "1" ]; then + return 1 + fi + + if [ ! -f ${lib.escapeShellArg setupFingerprintFile} ]; then + return 1 + fi + + _setup_cached_fingerprint=$(cat ${lib.escapeShellArg setupFingerprintFile} 2>/dev/null || echo "") + if [ "$_setup_current_fingerprint" != "$_setup_cached_fingerprint" ]; then + return 1 + fi + + if [ -z "${setupInnerCacheDirList}" ]; then + return 0 + fi + + for _setup_cache_dir_name in ${setupInnerCacheDirList}; do + _setup_cache_dir=${lib.escapeShellArg cache.cacheRoot}/$_setup_cache_dir_name + set -- "$_setup_cache_dir"/*.hash + if [ -f "$1" ]; then + return 0 + fi + done + + return 1 + } + ''; + setupTraceEnv = '' + if [ -n "''${DEVENV_TASK_OUTPUT_FILE:-}" ]; then + if [ -n "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ]; then + _root_trace=$(${pkgs.coreutils}/bin/od -An -tx1 -N16 /dev/urandom | tr -d ' \n') + _root_span=$(${pkgs.coreutils}/bin/od -An -tx1 -N8 /dev/urandom | tr -d ' \n') + _tp="00-''${_root_trace:0:32}-''${_root_span:0:16}-01" + _now_ns=$(${pkgs.coreutils}/bin/date +%s%N) + printf '{"devenv":{"env":{"DEVENV_SETUP_OUTER_CACHE_HIT":"%s","DEVENV_SETUP_FINGERPRINT":"%s","DEVENV_SETUP_GIT_HASH":"%s","TRACEPARENT":"%s","OTEL_SHELL_ENTRY_NS":"%s"}}}' \ + "$_setup_outer_cache_hit" \ + "$_setup_current_fingerprint" \ + "$_setup_git_hash" \ + "$_tp" \ + "$_now_ns" > "$DEVENV_TASK_OUTPUT_FILE" + else + printf '{"devenv":{"env":{"DEVENV_SETUP_OUTER_CACHE_HIT":"%s","DEVENV_SETUP_FINGERPRINT":"%s","DEVENV_SETUP_GIT_HASH":"%s"}}}' \ + "$_setup_outer_cache_hit" \ + "$_setup_current_fingerprint" \ + "$_setup_git_hash" > "$DEVENV_TASK_OUTPUT_FILE" + fi + fi + ''; in { tasks = cliGuard.stripGuards ( @@ -179,6 +307,9 @@ in "setup:gate" = lib.mkIf skipDuringRebase { description = "Check if setup should run (fails during rebase to skip setup)"; exec = '' + set -euo pipefail + ${setupFingerprintEnv} + _git_dir=$(${git} rev-parse --git-dir 2>/dev/null) if [ -d "$_git_dir/rebase-merge" ] || [ -d "$_git_dir/rebase-apply" ]; then echo "Skipping setup during git rebase/cherry-pick" @@ -186,28 +317,55 @@ in exit 1 fi - # Generate root trace context and propagate via devenv task output. - # Dependent tasks automatically receive TRACEPARENT + OTEL_SHELL_ENTRY_NS - # as env vars, linking all shell entry spans into a single trace. - if [ -n "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ] && [ -n "''${DEVENV_TASK_OUTPUT_FILE:-}" ]; then - _root_trace=$(${pkgs.coreutils}/bin/od -An -tx1 -N16 /dev/urandom | tr -d ' \n') - _root_span=$(${pkgs.coreutils}/bin/od -An -tx1 -N8 /dev/urandom | tr -d ' \n') - _tp="00-''${_root_trace:0:32}-''${_root_span:0:16}-01" - _now_ns=$(${pkgs.coreutils}/bin/date +%s%N) - printf '{"devenv":{"env":{"TRACEPARENT":"%s","OTEL_SHELL_ENTRY_NS":"%s"}}}' \ - "$_tp" "$_now_ns" > "$DEVENV_TASK_OUTPUT_FILE" + _setup_current_fingerprint="$(compute_setup_fingerprint)" + _setup_git_hash=$(${git} rev-parse HEAD 2>/dev/null || echo "no-git") + if setup_outer_cache_hit "$_setup_current_fingerprint"; then + _setup_outer_cache_hit="1" + else + _setup_outer_cache_hit="0" fi + + ${setupTraceEnv} ''; # This makes setup:gate run BEFORE each setup task # If gate fails, the tasks will be "skipped due to dependency failure" before = allSetupTasks; }; + "${setupRecordCacheTaskName}" = lib.mkIf (setupTasks != [ ]) { + description = "Record the successful setup fingerprint"; + after = lib.optionals skipDuringRebase [ "setup:gate" ] ++ setupTasks; + exec = '' + set -euo pipefail + ${setupFingerprintEnv} + + mkdir -p ${lib.escapeShellArg cache.cacheRoot} + + cache_value="''${DEVENV_SETUP_FINGERPRINT:-$(compute_setup_fingerprint)}" + ${cache.writeCacheFile ''"${setupFingerprintFile}"''} + + cache_value="''${DEVENV_SETUP_GIT_HASH:-$(${git} rev-parse HEAD 2>/dev/null || echo "no-git")}" + ${cache.writeCacheFile ''"${setupGitHashFile}"''} + ''; + status = '' + set -euo pipefail + if [ "''${FORCE_SETUP:-}" = "1" ]; then + exit 1 + fi + if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + exit 0 + fi + exit 1 + ''; + }; + # Wire setup tasks to run during shell entry. # Required tasks are hard dependencies; optional tasks use @completed so # failures don't block shell entry. "devenv:enterShell" = { - after = setupRequiredTasks ++ (map (t: "${t}@completed") setupOptionalTasks); + after = setupRequiredTasks + ++ (map (t: "${t}@completed") setupOptionalTasks) + ++ lib.optionals (setupTasks != [ ]) [ "${setupRecordCacheTaskName}@completed" ]; }; # Run setup tasks explicitly. diff --git a/nix/devenv-modules/tasks/shared/ts.nix b/nix/devenv-modules/tasks/shared/ts.nix index dfe7045e5..b8681afe3 100644 --- a/nix/devenv-modules/tasks/shared/ts.nix +++ b/nix/devenv-modules/tasks/shared/ts.nix @@ -53,6 +53,39 @@ let "genie:run" "pnpm:install" ]; + emitTsconfigHelper = '' + generate_emit_tsconfig() { + local source_tsconfig="$1" + local target_tsconfig="$2" + + ${pkgs.nodejs}/bin/node - "$source_tsconfig" "$target_tsconfig" <<'NODE' +const fs = require('node:fs') +const path = require('node:path') + +const [sourceTsconfig, targetTsconfig] = process.argv.slice(2) + +const readJsonWithLeadingComments = (filePath) => { + const contents = fs.readFileSync(filePath, 'utf8') + return JSON.parse(contents.replace(/^(?:\s*\/\/.*\n)+/, "")) +} + +const rootConfig = readJsonWithLeadingComments(sourceTsconfig) +const baseDir = path.dirname(sourceTsconfig) + +rootConfig.references = (rootConfig.references ?? []).filter((reference) => { + const refTsconfig = path.resolve(baseDir, reference.path, 'tsconfig.json') + if (!fs.existsSync(refTsconfig)) { + return true + } + + const refConfig = readJsonWithLeadingComments(refTsconfig) + return refConfig.compilerOptions?.noEmit !== true +}) + +fs.writeFileSync(targetTsconfig, JSON.stringify(rootConfig)) +NODE + } + ''; # Script that runs tsc with --extendedDiagnostics --verbose, # parses per-project timing, and emits OTEL child spans. @@ -213,11 +246,24 @@ let }; "ts:emit" = trace.withStatus "ts:emit" "binary" { description = "Emit build outputs without full type checking (tsc --build --noCheck)"; - exec = tscWithDiagnostics "--build ${tsconfigFile}" "--noCheck"; + exec = '' + set -euo pipefail + ${emitTsconfigHelper} + _emit_tmpdir="$(dirname "${tsconfigFile}")" + _emit_tsconfig="$(mktemp "$_emit_tmpdir/.ts-emit-XXXXXX.json")" + trap 'rm -f "$_emit_tsconfig"' EXIT + generate_emit_tsconfig "${tsconfigFile}" "$_emit_tsconfig" + ${tscWithDiagnostics "--build \"$_emit_tsconfig\"" "--noCheck"} + ''; status = '' set -euo pipefail + ${emitTsconfigHelper} - _out="$(${tscBin} --build ${tsconfigFile} --dry --noCheck --verbose --pretty false 2>&1)" || exit 1 + _emit_tmpdir="$(dirname "${tsconfigFile}")" + _emit_tsconfig="$(mktemp "$_emit_tmpdir/.ts-emit-XXXXXX.json")" + trap 'rm -f "$_emit_tsconfig"' EXIT + generate_emit_tsconfig "${tsconfigFile}" "$_emit_tsconfig" + _out="$(${tscBin} --build "$_emit_tsconfig" --dry --noCheck --verbose --pretty false 2>&1)" || exit 1 # tsc --build --dry reports pending work as: # - "A non-dry build would build project ..." # - "A non-dry build would update timestamps for output of project ..." From 7bbc70dbf6cb7636db8c2b847c3c34486c19bc22 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Fri, 27 Mar 2026 08:05:53 +0100 Subject: [PATCH 02/14] Document devenv perf cache logic --- nix/devenv-modules/tasks/shared/genie.nix | 2 ++ nix/devenv-modules/tasks/shared/pnpm.nix | 3 +++ nix/devenv-modules/tasks/shared/setup.nix | 8 ++++++++ nix/devenv-modules/tasks/shared/ts.nix | 7 +++++++ 4 files changed, 20 insertions(+) diff --git a/nix/devenv-modules/tasks/shared/genie.nix b/nix/devenv-modules/tasks/shared/genie.nix index 508cbfe8f..20fe96e77 100644 --- a/nix/devenv-modules/tasks/shared/genie.nix +++ b/nix/devenv-modules/tasks/shared/genie.nix @@ -24,6 +24,8 @@ let computeGenieStateHash = '' compute_genie_state_hash() { { + # Track both the `.genie.ts` sources and the generated files they own so + # warm status checks catch manual drift without booting the full CLI. ${pkgs.findutils}/bin/find . \ -type f \ -name '*.genie.ts' \ diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 34816e1f6..c4a5f0ec1 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -192,6 +192,9 @@ let computeProjectionStateHashFn = '' compute_projection_state_hash() { { + # Keep a cheap fingerprint for the realized node_modules projection. + # This catches missing/bad projections on the warm path without the + # deeper link-health scan that made cached installs expensive. for node_modules_dir in node_modules ${nodeModulesPaths}; do if [ -d "$node_modules_dir" ]; then printf 'dir %s\n' "$node_modules_dir" diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index 7e0a68837..12792ec48 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -157,6 +157,9 @@ let setupInnerCacheDirList = lib.concatMapStringsSep " " lib.escapeShellArg innerCacheDirs; setupFingerprintEnv = '' compute_setup_fingerprint() { + # Use git object IDs for tracked inputs and only content-hash dirty files. + # That keeps the warm-shell fingerprint cheap while still reacting to + # untracked/generated drift that git object IDs cannot describe. _setup_head=$(${git} rev-parse HEAD 2>/dev/null || echo "no-git") _setup_generated_from_head=$( ${git} grep -l -E '^// Source: .*\.genie\.ts|^# Source: .*\.genie\.ts' HEAD -- . 2>/dev/null || true @@ -237,6 +240,9 @@ let return 1 fi + # A matching outer fingerprint is only sufficient once at least one of the + # task-local caches exists again. This avoids skipping setup after users + # delete `.direnv/task-cache/*` without changing any tracked inputs. if [ -z "${setupInnerCacheDirList}" ]; then return 0 fi @@ -334,6 +340,8 @@ in "${setupRecordCacheTaskName}" = lib.mkIf (setupTasks != [ ]) { description = "Record the successful setup fingerprint"; + # Persist the outer cache only after the setup tasks finished. Writing it + # earlier would let later warm shells skip work that never completed. after = lib.optionals skipDuringRebase [ "setup:gate" ] ++ setupTasks; exec = '' set -euo pipefail diff --git a/nix/devenv-modules/tasks/shared/ts.nix b/nix/devenv-modules/tasks/shared/ts.nix index b8681afe3..85758c884 100644 --- a/nix/devenv-modules/tasks/shared/ts.nix +++ b/nix/devenv-modules/tasks/shared/ts.nix @@ -58,6 +58,9 @@ let local source_tsconfig="$1" local target_tsconfig="$2" + # `tsc --build --dry --noCheck` still treats `noEmit` references as emit + # work, which made `ts:emit` look perpetually stale. Build a filtered + # graph just for this task instead of mutating the checked-in config. ${pkgs.nodejs}/bin/node - "$source_tsconfig" "$target_tsconfig" <<'NODE' const fs = require('node:fs') const path = require('node:path') @@ -249,6 +252,8 @@ NODE exec = '' set -euo pipefail ${emitTsconfigHelper} + # Create the filtered config next to the source tsconfig so referenced + # project paths stay relative to the workspace instead of `/tmp`. _emit_tmpdir="$(dirname "${tsconfigFile}")" _emit_tsconfig="$(mktemp "$_emit_tmpdir/.ts-emit-XXXXXX.json")" trap 'rm -f "$_emit_tsconfig"' EXIT @@ -259,6 +264,8 @@ NODE set -euo pipefail ${emitTsconfigHelper} + # Reuse the same filtered graph for the dry-run status check so warm + # caching answers the same question as the real emit command. _emit_tmpdir="$(dirname "${tsconfigFile}")" _emit_tsconfig="$(mktemp "$_emit_tmpdir/.ts-emit-XXXXXX.json")" trap 'rm -f "$_emit_tsconfig"' EXIT From 538d5037c671d74424d58dde78825eb9fe80132c Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Fri, 27 Mar 2026 09:37:41 +0100 Subject: [PATCH 03/14] Use devenv task exports in setup gate --- nix/devenv-modules/tasks/shared/setup.nix | 49 +++++++++-------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index 12792ec48..56afa28c3 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -258,27 +258,6 @@ let return 1 } ''; - setupTraceEnv = '' - if [ -n "''${DEVENV_TASK_OUTPUT_FILE:-}" ]; then - if [ -n "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ]; then - _root_trace=$(${pkgs.coreutils}/bin/od -An -tx1 -N16 /dev/urandom | tr -d ' \n') - _root_span=$(${pkgs.coreutils}/bin/od -An -tx1 -N8 /dev/urandom | tr -d ' \n') - _tp="00-''${_root_trace:0:32}-''${_root_span:0:16}-01" - _now_ns=$(${pkgs.coreutils}/bin/date +%s%N) - printf '{"devenv":{"env":{"DEVENV_SETUP_OUTER_CACHE_HIT":"%s","DEVENV_SETUP_FINGERPRINT":"%s","DEVENV_SETUP_GIT_HASH":"%s","TRACEPARENT":"%s","OTEL_SHELL_ENTRY_NS":"%s"}}}' \ - "$_setup_outer_cache_hit" \ - "$_setup_current_fingerprint" \ - "$_setup_git_hash" \ - "$_tp" \ - "$_now_ns" > "$DEVENV_TASK_OUTPUT_FILE" - else - printf '{"devenv":{"env":{"DEVENV_SETUP_OUTER_CACHE_HIT":"%s","DEVENV_SETUP_FINGERPRINT":"%s","DEVENV_SETUP_GIT_HASH":"%s"}}}' \ - "$_setup_outer_cache_hit" \ - "$_setup_current_fingerprint" \ - "$_setup_git_hash" > "$DEVENV_TASK_OUTPUT_FILE" - fi - fi - ''; in { tasks = cliGuard.stripGuards ( @@ -303,15 +282,18 @@ in # Gate task that fails during rebase, causing dependent tasks to skip. # Uses `before` to inject itself as a dependency of each setup task. # - # OTEL trace propagation: - # Generates a W3C TRACEPARENT and propagates it to dependent tasks via - # devenv's native task output → env mechanism (devenv.env convention). - # When a task writes {"devenv":{"env":{"KEY":"VAL"}}} to $DEVENV_TASK_OUTPUT_FILE, - # devenv injects those as env vars into all subsequent task subprocesses. - # Ref: https://github.com/cachix/devenv/blob/main/devenv-tasks/src/task_state.rs#L134-L154 - # Ref: https://devenv.sh/tasks/ (Task Inputs and Outputs) + # The gate exports its computed cache metadata through devenv's native + # task export channel so every dependent status/exec sees the same + # `DEVENV_SETUP_*` values without re-running the fingerprint logic. "setup:gate" = lib.mkIf skipDuringRebase { description = "Check if setup should run (fails during rebase to skip setup)"; + exports = [ + "DEVENV_SETUP_OUTER_CACHE_HIT" + "DEVENV_SETUP_FINGERPRINT" + "DEVENV_SETUP_GIT_HASH" + "TRACEPARENT" + "OTEL_SHELL_ENTRY_NS" + ]; exec = '' set -euo pipefail ${setupFingerprintEnv} @@ -331,7 +313,16 @@ in _setup_outer_cache_hit="0" fi - ${setupTraceEnv} + export DEVENV_SETUP_OUTER_CACHE_HIT="$_setup_outer_cache_hit" + export DEVENV_SETUP_FINGERPRINT="$_setup_current_fingerprint" + export DEVENV_SETUP_GIT_HASH="$_setup_git_hash" + + if [ -n "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ]; then + _root_trace=$(${pkgs.coreutils}/bin/od -An -tx1 -N16 /dev/urandom | tr -d ' \n') + _root_span=$(${pkgs.coreutils}/bin/od -An -tx1 -N8 /dev/urandom | tr -d ' \n') + export TRACEPARENT="00-''${_root_trace:0:32}-''${_root_span:0:16}-01" + export OTEL_SHELL_ENTRY_NS="$(${pkgs.coreutils}/bin/date +%s%N)" + fi ''; # This makes setup:gate run BEFORE each setup task # If gate fails, the tasks will be "skipped due to dependency failure" From b3985f5b5975354327b85802de1459e68949ec1b Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Fri, 27 Mar 2026 09:49:23 +0100 Subject: [PATCH 04/14] Document upstream setup-cache constraints --- nix/devenv-modules/tasks/shared/setup.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index 56afa28c3..9c6219a63 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -157,6 +157,22 @@ let setupInnerCacheDirList = lib.concatMapStringsSep " " lib.escapeShellArg innerCacheDirs; setupFingerprintEnv = '' compute_setup_fingerprint() { + # This outer fingerprint exists because devenv's built-in `status` + # semantics do not prune a dependency subtree: the scheduler only runs a + # task's status command once that task itself is ready to execute, after + # its upstream dependencies have already been traversed. A cached + # aggregate `devenv:enterShell` task therefore would not avoid the warm + # `pnpm:install` / `genie:run` / `mr:apply` status probes we are trying + # to skip. + # + # We also intentionally avoid `execIfModified` here. Upstream has fixed + # several correctness and performance bugs in that path, but it is still + # the wrong primitive for repo bootstrap: setup invalidation depends on + # generated-file drift, lockfile topology, and shell/task exports rather + # than just a watched file set. Relevant upstream history: + # - #1924: enterShell + execIfModified caching was confusing for exports + # - #2422 / #2469 / #2588: glob walking could explode through node_modules + # - #2577: deletion/removal invalidation needed fixes # Use git object IDs for tracked inputs and only content-hash dirty files. # That keeps the warm-shell fingerprint cheap while still reacting to # untracked/generated drift that git object IDs cannot describe. @@ -285,6 +301,8 @@ in # The gate exports its computed cache metadata through devenv's native # task export channel so every dependent status/exec sees the same # `DEVENV_SETUP_*` values without re-running the fingerprint logic. + # This keeps us aligned with upstream task plumbing instead of carrying a + # parallel ad-hoc output protocol in this repo. "setup:gate" = lib.mkIf skipDuringRebase { description = "Check if setup should run (fails during rebase to skip setup)"; exports = [ From 85bda5e0f7fe65c6d0f4843b6885726ade8b1a2e Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Fri, 27 Mar 2026 10:17:45 +0100 Subject: [PATCH 05/14] Align pnpm smoke fixture with projection cache --- .../tasks/shared/tests/pnpm-task-smoke.test.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index 6234ccf96..ca2242046 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -159,6 +159,16 @@ if [ "${1:-}" = "install" ]; then fi mkdir -p node_modules touch node_modules/.install-ok + # The warm-path status now fingerprints the root projection metadata that + # pnpm always writes on a real install. Keep the smoke fixture aligned with + # that contract so the test still exercises the task logic instead of + # failing on an unrealistically incomplete fake install. + cat > node_modules/.modules.yaml <<'YAML' +hoistPattern: [] +nodeLinker: isolated +storeDir: /tmp/fake-pnpm-store +virtualStoreDir: node_modules/.pnpm +YAML exit 0 fi echo "unexpected fake pnpm invocation: $*" >&2 @@ -259,7 +269,9 @@ echo "Test 2: exec runs fake pnpm and populates cache" : > "$tmpdir/flock.log" bash "$tmpdir/pnpm-install.exec.sh" test -f "$workspace/.direnv/task-cache/pnpm-install/install-state.hash" + test -f "$workspace/.direnv/task-cache/pnpm-install/projection-state.hash" test -d "$workspace/node_modules" + test -f "$workspace/node_modules/.modules.yaml" grep -qxF "flock -w 600 200" "$tmpdir/flock.log" grep -qxF "flock -w 600 201" "$tmpdir/flock.log" grep -qxF "flock -w 600 202" "$tmpdir/flock.log" From bb7fc8681e6d9dd1a226eaa6c4be01a3544efdd4 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Fri, 27 Mar 2026 11:55:31 +0100 Subject: [PATCH 06/14] Harden devenv setup fast paths --- CHANGELOG.md | 1 + nix/devenv-modules/tasks/shared/genie.nix | 39 +++- nix/devenv-modules/tasks/shared/pnpm.nix | 14 +- nix/devenv-modules/tasks/shared/setup.nix | 36 ++-- .../shared/tests/pnpm-task-smoke.test.sh | 48 +++-- .../tasks/shared/tests/setup-cache.test.sh | 181 ++++-------------- nix/devenv-modules/tasks/shared/ts.nix | 7 +- 7 files changed, 132 insertions(+), 194 deletions(-) mode change 100755 => 100644 nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab710b84..bc50a7475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ All notable changes to this project will be documented in this file. - Adds an outer `setup:auto` cache so warm `devenv shell` skips unchanged bootstrap work instead of traversing `pnpm:install`, `genie:run`, and `mr:apply` on every entry - Switches shell bootstrap from `mr:sync` to initial `mr:apply` so a fresh worktree is normalized without fetching on every shell - Speeds up warm task status paths by using direct `mr status`, fingerprint-based `genie:run` caching, lighter `pnpm:install` status checks, and a `ts:emit` graph that excludes `noEmit` references at emit time + - Hardens the fast paths by making the outer cache only track setup inputs while each task still verifies its own outputs before skipping - **@overeng/genie**: Validate GitHub Actions `runs-on` labels before emitting workflow YAML - Fails `genie` when workflow jobs serialize non-string, empty, or stale placeholder runner labels like `null` / `...=undefined` - Prevents CI helper API drift from silently generating invalid workflow files that only fail later in GitHub Actions diff --git a/nix/devenv-modules/tasks/shared/genie.nix b/nix/devenv-modules/tasks/shared/genie.nix index 20fe96e77..df348b7d2 100644 --- a/nix/devenv-modules/tasks/shared/genie.nix +++ b/nix/devenv-modules/tasks/shared/genie.nix @@ -21,9 +21,26 @@ let }; cacheRoot = ".direnv/task-cache/genie-run"; stateFile = "${cacheRoot}/state.hash"; + generatedFilesFile = "${cacheRoot}/generated-files.txt"; + collectGenieGeneratedFiles = '' + collect_genie_generated_files() { + ${pkgs.ripgrep}/bin/rg -l \ + --glob '!tmp/**' \ + --glob '!.git/**' \ + --glob '!.direnv/**' \ + --glob '!.devenv/**' \ + --glob '!node_modules/**' \ + '^// Source: .*\.genie\.ts|^# Source: .*\.genie\.ts' . || true + } + ''; computeGenieStateHash = '' compute_genie_state_hash() { { + if command -v genie >/dev/null 2>&1; then + printf 'genie-path %s\n' "$(command -v genie)" + printf 'genie-version %s\n' "$(genie --version 2>/dev/null | ${pkgs.coreutils}/bin/head -n1 || echo unknown)" + fi + # Track both the `.genie.ts` sources and the generated files they own so # warm status checks catch manual drift without booting the full CLI. ${pkgs.findutils}/bin/find . \ @@ -34,13 +51,7 @@ let -not -path './.devenv/*' \ -not -path './node_modules/*' \ -print - ${pkgs.ripgrep}/bin/rg -l \ - --glob '!tmp/**' \ - --glob '!.git/**' \ - --glob '!.direnv/**' \ - --glob '!.devenv/**' \ - --glob '!node_modules/**' \ - '^// Source: .*\.genie\.ts|^# Source: .*\.genie\.ts' . + ${collectGenieGeneratedFiles} } \ | LC_ALL=C sort -u \ | while IFS= read -r file; do @@ -67,6 +78,7 @@ let exec = trace.exec "genie:run" '' set -euo pipefail mkdir -p ${lib.escapeShellArg cacheRoot} + ${collectGenieGeneratedFiles} ${computeGenieStateHash} genie cache_value="$(compute_genie_state_hash)" @@ -77,10 +89,23 @@ let else mv "$tmp_file" ${lib.escapeShellArg stateFile} fi + + generated_tmp_file="$(mktemp)" + collect_genie_generated_files | LC_ALL=C sort -u > "$generated_tmp_file" + mv "$generated_tmp_file" ${lib.escapeShellArg generatedFilesFile} ''; status = trace.status "genie:run" "binary" '' set -euo pipefail if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + # The outer setup fingerprint already covers tracked generated-file + # drift plus genie binary identity. On that warm path, only prove that + # the outputs we generated last time still exist. + [ -f ${lib.escapeShellArg stateFile} ] || exit 1 + [ -f ${lib.escapeShellArg generatedFilesFile} ] || exit 1 + while IFS= read -r file; do + [ -n "$file" ] || continue + [ -f "$file" ] || exit 1 + done < ${lib.escapeShellArg generatedFilesFile} exit 0 fi [ -f ${lib.escapeShellArg stateFile} ] || exit 1 diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index c4a5f0ec1..652b596d0 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -182,6 +182,7 @@ let gvs_links_dir="$(resolve_gvs_links_dir)" { + printf '%s\n' ${lib.escapeShellArg pkgs.pnpm.version} printf '%s\n' "$workspace_state_hash" printf '%s\n' "''${gvs_links_dir:-}" printf '%s\n' ${lib.escapeShellArg (builtins.toJSON installFlags)} @@ -423,9 +424,6 @@ let status = trace.status installTaskName "hash" '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} - if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then - exit 0 - fi ${loadPnpmTaskHelpersFn} ${ensureLocalPnpmHomeFn} ${ensureLocalPnpmStoreDirFn} @@ -436,6 +434,16 @@ let exit 1 fi + if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + ${computeProjectionStateHashFn} + current_projection_hash="$(compute_projection_state_hash)" + stored_projection_hash="$(cat "$projection_hash_file")" + if [ "$current_projection_hash" != "$stored_projection_hash" ]; then + exit 1 + fi + exit 0 + fi + ${computeWorkspaceStateHash} ${computeInstallStateHashFn} ${computeProjectionStateHashFn} diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index 9c6219a63..21db76dbc 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -8,7 +8,7 @@ # imports = [ # (taskModules.setup { # requiredTasks = [ ]; -# optionalTasks = [ "pnpm:install" "genie:run" "ts:emit" ]; +# optionalTasks = [ "pnpm:install" "genie:run" "mr:apply" ]; # completionsCliNames = [ "genie" "mr" ]; # }) # ]; @@ -30,7 +30,6 @@ requiredTasks ? [ ], optionalTasks ? [ ], completionsCliNames ? [ ], - innerCacheDirs ? [ ], skipDuringRebase ? true, }: { @@ -101,10 +100,6 @@ let exit 0 ''; completionsStatus = '' - if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then - exit 0 - fi - shell="" if [ -n "''${FISH_VERSION:-}" ]; then shell="fish" @@ -154,7 +149,6 @@ let setupOptionalTasks = userOptionalTasks ++ lib.optionals completionsEnabled [ completionsTaskName ]; setupTasks = setupRequiredTasks ++ setupOptionalTasks; allSetupTasks = setupTasks; - setupInnerCacheDirList = lib.concatMapStringsSep " " lib.escapeShellArg innerCacheDirs; setupFingerprintEnv = '' compute_setup_fingerprint() { # This outer fingerprint exists because devenv's built-in `status` @@ -212,6 +206,17 @@ let { printf 'head %s\n' "$_setup_head" + # Shell-entry tasks can short-circuit to lightweight output checks once + # the repo inputs are unchanged. Include the task tool identities here so + # changing the active pnpm/genie/mr binary still invalidates the outer + # cache and forces the next shell to re-validate or refresh setup. + for _setup_tool in pnpm genie mr; do + if command -v "$_setup_tool" >/dev/null 2>&1; then + printf 'tool %s path %s\n' "$_setup_tool" "$(command -v "$_setup_tool")" + printf 'tool %s version %s\n' "$_setup_tool" "$($_setup_tool --version 2>/dev/null | ${pkgs.coreutils}/bin/head -n1 || echo unknown)" + fi + done + for _setup_file in package.json pnpm-workspace.yaml pnpm-lock.yaml .npmrc megarepo.kdl megarepo.json megarepo.lock; do ${git} ls-files -s -- "$_setup_file" 2>/dev/null || true done @@ -256,22 +261,7 @@ let return 1 fi - # A matching outer fingerprint is only sufficient once at least one of the - # task-local caches exists again. This avoids skipping setup after users - # delete `.direnv/task-cache/*` without changing any tracked inputs. - if [ -z "${setupInnerCacheDirList}" ]; then - return 0 - fi - - for _setup_cache_dir_name in ${setupInnerCacheDirList}; do - _setup_cache_dir=${lib.escapeShellArg cache.cacheRoot}/$_setup_cache_dir_name - set -- "$_setup_cache_dir"/*.hash - if [ -f "$1" ]; then - return 0 - fi - done - - return 1 + return 0 } ''; in diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index ca2242046..ffca45b21 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -292,7 +292,31 @@ echo "Test 3: status hits after install with same GVS path" assert_exit_code 0 "$exit_code" "status should hit after install" ) -echo "Test 4: exec defaults PNPM_HOME to a workspace-local projection" +echo "Test 4: outer cache hit still misses when projection metadata is missing" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + export DEVENV_SETUP_OUTER_CACHE_HIT=1 + rm -f node_modules/.modules.yaml + set +e + bash "$tmpdir/pnpm-install.status.sh" + exit_code=$? + set -e + unset DEVENV_SETUP_OUTER_CACHE_HIT + assert_exit_code 1 "$exit_code" "outer-hit status should miss when .modules.yaml is missing" +) + +echo "Test 5: exec restores projection metadata after a miss" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + bash "$tmpdir/pnpm-install.exec.sh" + test -f "$workspace/node_modules/.modules.yaml" +) + +echo "Test 6: exec defaults PNPM_HOME to a workspace-local projection" ( cd "$workspace" export HOME="$tmpdir/home" @@ -304,7 +328,7 @@ echo "Test 4: exec defaults PNPM_HOME to a workspace-local projection" grep -qxF "npm_config_store_dir=$workspace/.direnv/pnpm-store" "$tmpdir/pnpm.log" ) -echo "Test 5: status hits after install with the default GVS path" +echo "Test 7: status hits after install with the default GVS path" ( cd "$workspace" export HOME="$tmpdir/home" @@ -316,7 +340,7 @@ echo "Test 5: status hits after install with the default GVS path" assert_exit_code 0 "$exit_code" "status should hit after default-PNPM_HOME install" ) -echo "Test 6: status still hits when PNPM_HOME changes but store-dir stays shared" +echo "Test 8: status still hits when PNPM_HOME changes but store-dir stays shared" ( cd "$workspace" export HOME="$tmpdir/home" @@ -328,7 +352,7 @@ echo "Test 6: status still hits when PNPM_HOME changes but store-dir stays share assert_exit_code 0 "$exit_code" "status should hit when only PNPM_HOME changes" ) -echo "Test 7: status misses after effective store-dir changes" +echo "Test 9: status misses after effective store-dir changes" ( cd "$workspace" export HOME="$tmpdir/home" @@ -341,10 +365,10 @@ echo "Test 7: status misses after effective store-dir changes" assert_exit_code 1 "$exit_code" "status should miss when store-dir changes" ) -echo "Test 8: exec invoked pnpm install" +echo "Test 10: exec invoked pnpm install" grep -q "^install " "$tmpdir/pnpm.log" -echo "Test 9: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" +echo "Test 11: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" ( cd "$workspace" export HOME="$tmpdir/home" @@ -361,7 +385,7 @@ echo "Test 9: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shar grep -qxF "npm_config_store_dir=$workspace/.direnv/pnpm-store" "$tmpdir/pnpm.log" ) -echo "Test 10: nested workspace status hits after nested install" +echo "Test 12: nested workspace status hits after nested install" ( cd "$workspace" export HOME="$tmpdir/home" @@ -375,7 +399,7 @@ echo "Test 10: nested workspace status hits after nested install" assert_exit_code 0 "$exit_code" "nested status should hit after nested install" ) -echo "Test 11: install flags and pre-install hooks are applied" +echo "Test 13: install flags and pre-install hooks are applied" ( cd "$workspace" export HOME="$tmpdir/home" @@ -389,7 +413,7 @@ echo "Test 11: install flags and pre-install hooks are applied" grep -qxF "install --config.confirmModulesPurge=false --config.store-dir=$workspace/.direnv/pnpm-store --ignore-scripts --config.public-hoist-pattern=*" "$tmpdir/pnpm.log" ) -echo "Test 12: CI install failures preserve and classify the pnpm log" +echo "Test 14: CI install failures preserve and classify the pnpm log" ( cd "$workspace" export HOME="$tmpdir/home" @@ -413,21 +437,21 @@ echo "Test 12: CI install failures preserve and classify the pnpm log" grep -qF "Socket timeout" <<< "$output" ) -echo "Test 13: generated test task runs vitest without pnpm exec" +echo "Test 15: generated test task runs vitest without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/test-demo.exec.sh")" [ "$output" = "vitest-shim:run" ] ) -echo "Test 14: generated storybook task runs storybook without pnpm exec" +echo "Test 16: generated storybook task runs storybook without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/storybook-demo.exec.sh")" [ "$output" = "storybook-shim:build" ] ) -echo "Test 15: clean leaves shared GVS links intact" +echo "Test 17: clean leaves shared GVS links intact" ( cd "$workspace" mkdir -p "$workspace/.direnv/pnpm-store/v11/links/shared-pkg" diff --git a/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh b/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh old mode 100755 new mode 100644 index a8ee300fa..d9d65171a --- a/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh @@ -1,18 +1,11 @@ #!/usr/bin/env bash -# Tests for setup.nix git hash caching with inner cache awareness +# Tests for setup.nix outer setup fingerprint caching. # -# Validates the two-tier caching design (R5, R11 compliance): -# - Outer tier: git hash -# - Inner tier: per-task content caches (e.g., pnpm-install/*.hash) -# -# Tasks should only be skipped when BOTH tiers are valid. -# If innerCacheDirs is empty, inner cache check is skipped (git-hash-only mode). +# The outer cache only answers whether shell-entry inputs changed. Task-local +# status checks own output validation, so this test intentionally stays focused +# on fingerprint persistence and FORCE_SETUP behavior. set -euo pipefail -# ============================================================================ -# Test helpers -# ============================================================================ - assert_exit_code() { local expected="$1" local actual="$2" @@ -27,194 +20,86 @@ assert_exit_code() { echo " ok: $label" } -# ============================================================================ -# Simulate the gitHashStatus function from setup.nix -# This mirrors the logic so we can test it in isolation -# ============================================================================ - -simulate_git_hash_status() { - local hash_file="$1" - local cache_root="$2" - local current_hash="$3" - local force_setup="${4-}" # Explicit parameter only, ignore env var for testing - local inner_cache_dirs="${5-pnpm-install}" # space-separated list, empty = git-hash-only +simulate_setup_outer_cache_hit() { + local fingerprint_file="$1" + local current_fingerprint="$2" + local force_setup="${3-}" # Explicit parameter only, ignore env var for testing - # Allow bypass via force_setup parameter (NOT env var - env var breaks CI tests) [ "$force_setup" = "1" ] && return 1 local cached - cached=$(cat "$hash_file" 2>/dev/null || echo "") - - # If git hash differs, always run - if [ "$current_hash" != "$cached" ]; then - return 1 - fi - - # If no inner cache dirs configured, use git-hash-only mode - if [ -z "$inner_cache_dirs" ]; then - return 0 - fi - - # Check each configured inner cache dir for *.hash files - for dir_name in $inner_cache_dirs; do - local cache_dir="$cache_root/$dir_name" - # Directory must exist and contain at least one .hash file - if [ -d "$cache_dir" ]; then - # Simple and reliable: iterate over files and check suffix - for f in "$cache_dir"/*; do - case "$f" in - *.hash) - [ -f "$f" ] && return 0 - ;; - esac - done - fi - done - - # No valid inner caches found - run to populate them - return 1 + cached=$(cat "$fingerprint_file" 2>/dev/null || echo "") + [ "$current_fingerprint" = "$cached" ] } -# ============================================================================ -# Test cases -# ============================================================================ - echo "Running setup-cache tests..." echo "" -# Create temp directory structure test_dir=$(mktemp -d) trap 'rm -rf "$test_dir"' EXIT cache_root="$test_dir/.direnv/task-cache" -hash_file="$cache_root/setup-git-hash" -pnpm_cache_dir="$cache_root/pnpm-install" +fingerprint_file="$cache_root/setup-fingerprint" mkdir -p "$cache_root" -# Test 1: Fresh cache (no git hash file) -> should return 1 (run) -echo "Test 1: Fresh cache (no git hash file)" +echo "Test 1: Fresh cache (no fingerprint file)" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" +simulate_setup_outer_cache_hit "$fingerprint_file" "abc123" exit_code=$? set -e assert_exit_code 1 "$exit_code" "fresh cache returns 1 (needs to run)" -# Test 2: Matching git hash but NO inner caches -> should return 1 (run) echo "" -echo "Test 2: Matching git hash but no inner caches" -echo "abc123" > "$hash_file" +echo "Test 2: Matching fingerprint" +echo "abc123" > "$fingerprint_file" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" +simulate_setup_outer_cache_hit "$fingerprint_file" "abc123" exit_code=$? set -e -assert_exit_code 1 "$exit_code" "matching hash + no inner caches returns 1 (run to populate)" +assert_exit_code 0 "$exit_code" "matching fingerprint returns 0 (skip)" -# Test 3: Matching git hash AND inner caches with .hash files -> should return 0 (skip) echo "" -echo "Test 3: Matching git hash + inner caches with .hash files" -mkdir -p "$pnpm_cache_dir" -echo "somehash" > "$pnpm_cache_dir/genie.hash" +echo "Test 3: Different fingerprint" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" +simulate_setup_outer_cache_hit "$fingerprint_file" "def456" exit_code=$? set -e -assert_exit_code 0 "$exit_code" "matching hash + .hash files returns 0 (skip)" +assert_exit_code 1 "$exit_code" "different fingerprint returns 1 (needs to run)" -# Test 4: Different git hash -> should return 1 (run) even with inner caches echo "" -echo "Test 4: Different git hash (inner caches exist)" +echo "Test 4: FORCE_SETUP=1 bypasses cache" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "def456" -exit_code=$? -set -e -assert_exit_code 1 "$exit_code" "different hash returns 1 (needs to run)" - -# Test 5: FORCE_SETUP=1 -> should return 1 (run) regardless of cache state -echo "" -echo "Test 5: FORCE_SETUP=1 bypasses cache" -set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" "1" +simulate_setup_outer_cache_hit "$fingerprint_file" "abc123" "1" exit_code=$? set -e assert_exit_code 1 "$exit_code" "FORCE_SETUP=1 returns 1 (always run)" -# Test 6: Empty inner cache directory -> should return 1 (run) -echo "" -echo "Test 6: Empty inner cache directory" -rm -f "$pnpm_cache_dir"/* -set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" -exit_code=$? -set -e -assert_exit_code 1 "$exit_code" "empty inner cache dir returns 1 (run to populate)" - -# Test 7: Inner cache with multiple .hash files -> should return 0 (skip) -echo "" -echo "Test 7: Multiple inner cache .hash files" -echo "hash1" > "$pnpm_cache_dir/genie.hash" -echo "hash2" > "$pnpm_cache_dir/megarepo.hash" -echo "hash3" > "$pnpm_cache_dir/utils.hash" -set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" -exit_code=$? -set -e -assert_exit_code 0 "$exit_code" "multiple .hash files returns 0 (skip)" - -# Test 8: Inner cache with only non-.hash files -> should return 1 (run) -echo "" -echo "Test 8: Inner cache with only non-.hash files (false positive prevention)" -rm -f "$pnpm_cache_dir"/* -echo "not a hash" > "$pnpm_cache_dir/some.lock" -echo "also not" > "$pnpm_cache_dir/partial.tmp" -set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" -exit_code=$? -set -e -assert_exit_code 1 "$exit_code" "non-.hash files returns 1 (run to populate proper caches)" - -# Test 9: Git-hash-only mode (empty innerCacheDirs) -> should return 0 when hash matches -echo "" -echo "Test 9: Git-hash-only mode (innerCacheDirs='')" -rm -rf "$pnpm_cache_dir" # Remove inner caches entirely -set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" "" "" -exit_code=$? -set -e -assert_exit_code 0 "$exit_code" "git-hash-only mode returns 0 when hash matches" - -# Test 10: Git-hash-only mode with different hash -> should return 1 (run) echo "" -echo "Test 10: Git-hash-only mode with different hash" +echo "Test 5: Empty fingerprint file" +: > "$fingerprint_file" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "xyz999" "" "" +simulate_setup_outer_cache_hit "$fingerprint_file" "abc123" exit_code=$? set -e -assert_exit_code 1 "$exit_code" "git-hash-only mode returns 1 when hash differs" +assert_exit_code 1 "$exit_code" "empty fingerprint file returns 1 (needs to run)" -# Test 11: Multiple inner cache dirs, only one has .hash files -> should return 0 (skip) echo "" -echo "Test 11: Multiple inner cache dirs, partial population" -mkdir -p "$pnpm_cache_dir" -mkdir -p "$cache_root/other-cache" -echo "hash1" > "$pnpm_cache_dir/genie.hash" -# other-cache has no .hash files +echo "Test 6: Trailing newline in cache file still matches" +printf 'abc123\n' > "$fingerprint_file" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" "" "pnpm-install other-cache" +simulate_setup_outer_cache_hit "$fingerprint_file" "abc123" exit_code=$? set -e -assert_exit_code 0 "$exit_code" "at least one valid inner cache returns 0 (skip)" +assert_exit_code 0 "$exit_code" "cached newline-trimmed fingerprint returns 0 (skip)" -# Test 12: Multiple inner cache dirs, none have .hash files -> should return 1 (run) echo "" -echo "Test 12: Multiple inner cache dirs, none populated" -rm -f "$pnpm_cache_dir"/*.hash -echo "not a hash" > "$cache_root/other-cache/lock.file" +echo "Test 7: Similar but different fingerprint does not false-hit" set +e -simulate_git_hash_status "$hash_file" "$cache_root" "abc123" "" "pnpm-install other-cache" +simulate_setup_outer_cache_hit "$fingerprint_file" "abc1234" exit_code=$? set -e -assert_exit_code 1 "$exit_code" "no valid inner caches returns 1 (run)" +assert_exit_code 1 "$exit_code" "different fingerprint text returns 1 (needs to run)" echo "" echo "All setup-cache tests passed" diff --git a/nix/devenv-modules/tasks/shared/ts.nix b/nix/devenv-modules/tasks/shared/ts.nix index 85758c884..016987ce3 100644 --- a/nix/devenv-modules/tasks/shared/ts.nix +++ b/nix/devenv-modules/tasks/shared/ts.nix @@ -72,11 +72,16 @@ const readJsonWithLeadingComments = (filePath) => { return JSON.parse(contents.replace(/^(?:\s*\/\/.*\n)+/, "")) } +const resolveReferenceTsconfig = (referencePath) => { + const resolvedPath = path.resolve(baseDir, referencePath) + return path.extname(resolvedPath) ? resolvedPath : path.join(resolvedPath, 'tsconfig.json') +} + const rootConfig = readJsonWithLeadingComments(sourceTsconfig) const baseDir = path.dirname(sourceTsconfig) rootConfig.references = (rootConfig.references ?? []).filter((reference) => { - const refTsconfig = path.resolve(baseDir, reference.path, 'tsconfig.json') + const refTsconfig = resolveReferenceTsconfig(reference.path) if (!fs.existsSync(refTsconfig)) { return true } From fcf462bd305e5ac9d43442ac1f5d31025b4672ab Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Fri, 27 Mar 2026 11:55:46 +0100 Subject: [PATCH 07/14] Restore setup-cache test mode --- nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh diff --git a/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh b/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh old mode 100644 new mode 100755 From dbbd9d4f9a9554a7a07740c6191c7f04ef2b4b71 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Sat, 28 Mar 2026 16:03:51 +0100 Subject: [PATCH 08/14] Pin devenv for OTEL shell-entry validation --- .github/workflows/ci.yml | 103 +++++++ .github/workflows/ci.yml.genie.ts | 27 +- CHANGELOG.md | 4 + context/otel.md | 4 +- context/workarounds/devenv-issues.md | 19 +- devenv.lock | 8 +- devenv.yaml | 5 +- genie/ci-workflow.ts | 4 + nix/devenv-modules/otel.nix | 281 +++++++++++------- nix/devenv-modules/tasks/shared/pnpm.nix | 18 +- .../shared/tests/pnpm-task-smoke.test.sh | 41 ++- 11 files changed, 380 insertions(+), 134 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f788ac04..fd018e24a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,6 +184,109 @@ jobs: EOF echo "::warning::Intentional failure for diagnostics validation (#272)" exit 1 + - name: Verify OTEL shell entry + shell: bash + run: | + __nix_gc_retry() { + local __task='devenv tasks run otel:test --mode before' __max=${NIX_GC_RACE_MAX_RETRIES:-10} __heartbeat=${CI_PROGRESS_HEARTBEAT_SECONDS:-60} __n=1 __log __rc __path __start __now __elapsed __hb_pid __flattened __saw_invalid_path __saw_cachix_signature + __start=$(date +%s) + + __write_summary() { + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || return 0 + { + echo "### CI Task" + # Keep summary values plain text. Backticks inside double quotes trigger + # shell command substitution and turned failed-task metadata into bogus + # commands on GitHub Actions runners. + echo "- Task: $__task" + echo "- Status: $1" + echo "- Duration: $__elapsed s" + echo "- Attempts: $__n/$__max" + [ -z "${2:-}" ] || echo "- Note: $2" + } >> "$GITHUB_STEP_SUMMARY" + } + + while [ "$__n" -le "$__max" ]; do + echo "::notice::[ci] starting $__task (attempt $__n/$__max)" + ( + while sleep "$__heartbeat"; do + __now=$(date +%s) + __elapsed=$((__now - __start)) + echo "::notice::[ci] $__task still running after $__elapsed s (attempt $__n/$__max)" + done + ) & + __hb_pid=$! + + __log=$(mktemp) + set +e + eval "$1" > >(tee -a "$__log") 2> >(tee -a "$__log" >&2) + __rc=$? + set -e + + kill "$__hb_pid" 2>/dev/null || true + wait "$__hb_pid" 2>/dev/null || true + + __now=$(date +%s) + __elapsed=$((__now - __start)) + + [ $__rc -eq 0 ] && { + echo "::notice::[ci] completed $__task in $__elapsed s" + if [ "$__n" -gt 1 ]; then + __write_summary success "Recovered from Nix GC race after retry" + else + __write_summary success + fi + rm -f "$__log" + return 0 + } + + __flattened=$(perl -0pe 's/\e\[[0-9;]*m//g; s/\n/ /g' "$__log") + __path=$(printf '%s' "$__flattened" | grep -oP "error:\s+path '\K/nix/store/[^']*(?='\s+is not valid)" 2>/dev/null | head -1 | tr -d '[:space:]' || true) + __saw_invalid_path=false + __saw_cachix_signature=false + [ -n "$__path" ] && __saw_invalid_path=true + printf '%s' "$__flattened" | grep -q 'Failed to convert config\.cachix to JSON' && __saw_cachix_signature=true || true + # Match the semantic signal, not the exact quote punctuation, so the shell + # stays valid even when the human-facing error wraps the option name. + printf '%s' "$__flattened" | grep -q 'while evaluating the option' && printf '%s' "$__flattened" | grep -q 'cachix\.package' && __saw_cachix_signature=true || true + rm -f "$__log" + if [ "$__saw_invalid_path" != true ] && [ "$__saw_cachix_signature" != true ]; then + echo "::warning::[ci] $__task failed after $__elapsed s without a detected Nix store validity race" + __write_summary failure "No Nix GC race signature detected" + return $__rc + fi + if [ "$__saw_cachix_signature" = true ] && [ -n "$__path" ]; then + echo "::warning::Nix store validity race detected for $__task via cachix eval wrapper (attempt $__n/$__max): $__path" + elif [ "$__saw_cachix_signature" = true ]; then + # The cachix wrapper can surface the GC race before the invalid path makes + # it into the flattened log. Retrying after clearing the eval cache still + # recovers that case in practice. + echo "::warning::Nix store validity race detected for $__task via cachix eval wrapper without extracted store path (attempt $__n/$__max)" + else + echo "::warning::Nix store validity race detected for $__task (attempt $__n/$__max): $__path" + fi + [ -z "$__path" ] || nix-store --realise "$__path" 2>/dev/null || true + rm -rf ~/.cache/nix/eval-cache-* + __n=$((__n + 1)) + done + + __now=$(date +%s) + __elapsed=$((__now - __start)) + echo "::error::Nix GC race retry exhausted for $__task ($__max attempts)" + __write_summary failure "Nix GC race retry exhausted" + return 1 + }; __nix_gc_retry 'if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" PNPM_HOME="${PNPM_HOME:-${{ github.workspace }}/.pnpm-home}" PNPM_STORE_DIR="${PNPM_STORE_DIR:-${{ runner.temp }}/pnpm-store/${{ github.job }}}" DT_PASSTHROUGH=1 "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run otel:test --mode before' + command -v script >/dev/null 2>&1 + tmp_log="$(mktemp)" + printf 'printf "OTEL_MODE=%%s\n" "$OTEL_MODE" + printf "OTEL_GRAFANA_LINK_URL=%%s\n" "$OTEL_GRAFANA_LINK_URL" + exit + ' | script -qefc '"${DEVENV_BIN:?DEVENV_BIN not set}" shell --no-reload' "$tmp_log" + grep -q '\[otel\] Using .* OTEL stack' "$tmp_log" + grep -q '\[otel\] Start with: devenv up' "$tmp_log" + grep -q '^OTEL_MODE=' "$tmp_log" + grep -q '^OTEL_GRAFANA_LINK_URL=http' "$tmp_log" + rm -f "$tmp_log" - name: Type check run: | __nix_gc_retry_helper=$(mktemp) diff --git a/.github/workflows/ci.yml.genie.ts b/.github/workflows/ci.yml.genie.ts index bfd07abc3..c4faf73f1 100644 --- a/.github/workflows/ci.yml.genie.ts +++ b/.github/workflows/ci.yml.genie.ts @@ -69,6 +69,28 @@ const failureReminderStep = { ].join('\n'), } as const +/** + * Verify the lock-pinned devenv rev emits OTEL shell-entry messages under a real PTY. + * `--no-reload` keeps the probe on the post-init shell-output path we care about + * without exercising the separate interactive reload loop, which currently + * panics on the pinned upstream commit. + */ +const verifyOtelShellEntryStep = { + name: 'Verify OTEL shell entry', + shell: 'bash' as const, + run: [ + runDevenvTasksBefore('otel:test'), + 'command -v script >/dev/null 2>&1', + 'tmp_log="$(mktemp)"', + `printf 'printf "OTEL_MODE=%%s\\n" "$OTEL_MODE"\nprintf "OTEL_GRAFANA_LINK_URL=%%s\\n" "$OTEL_GRAFANA_LINK_URL"\nexit\n' | script -qefc '"${'${DEVENV_BIN:?DEVENV_BIN not set}'}" shell --no-reload' "$tmp_log"`, + "grep -q '\\[otel\\] Using .* OTEL stack' \"$tmp_log\"", + "grep -q '\\[otel\\] Start with: devenv up' \"$tmp_log\"", + "grep -q '^OTEL_MODE=' \"$tmp_log\"", + "grep -q '^OTEL_GRAFANA_LINK_URL=http' \"$tmp_log\"", + 'rm -f "$tmp_log"', + ].join('\n'), +} as const + /** * Temporary diagnostics summary for #272. * Remove once #201/#272 are root-caused and we can return to a minimal CI flow. @@ -113,7 +135,7 @@ const nixDiagnosticsSummaryStep = { ].join('\n'), } as const -const job = (step: { name: string; run: string; shell?: string }) => ({ +const job = (step: { name: string; run: string }, extraSteps: readonly any[] = []) => ({ 'runs-on': namespaceRunner({ profile: 'namespace-profile-linux-x86-64', runId: '${{ github.run_id }}', @@ -122,6 +144,7 @@ const job = (step: { name: string; run: string; shell?: string }) => ({ env: standardCIEnv, steps: [ ...baseSteps, + ...extraSteps, step, savePnpmStateStep(), nixDiagnosticsSummaryStep, @@ -187,7 +210,7 @@ const jobs: Record | ReturnType Explore -> Tempo ``` @@ -93,7 +93,7 @@ otel-trace | cat # plain text: trace: The function parses `TRACEPARENT` (W3C format: `version-traceId-spanId-traceFlags`) and constructs a Grafana Explore URL from `OTEL_GRAFANA_LINK_URL`. -**Note:** Auto-display of the trace URL on shell entry is blocked by devenv's PTY task runner (`drain_pty_to_vt`), which consumes all shell output before the interactive session starts. Tracked upstream in [cachix/devenv#2500](https://github.com/cachix/devenv/issues/2500). +**Note:** This repo now uses `devenv.messages` to auto-display the OTEL shell-entry notice. `otel-trace` remains as an on-demand way to reopen the same link later in the session. The repo is temporarily pinned to the upstream post-[cachix/devenv#2661](https://github.com/cachix/devenv/pull/2661) commit while waiting for the next tagged release. ### `otel-span` -- Trace span CLI diff --git a/context/workarounds/devenv-issues.md b/context/workarounds/devenv-issues.md index cf9d4e259..ef1267b85 100644 --- a/context/workarounds/devenv-issues.md +++ b/context/workarounds/devenv-issues.md @@ -124,6 +124,10 @@ removed as they are no longer needed. **Issue:** https://github.com/cachix/devenv/issues/2500 +**Upstream status:** Fixed by https://github.com/cachix/devenv/pull/2661. + +**Repo status:** Temporarily resolved here by pinning `devenv` to the merged upstream commit while waiting for the next tagged release. + **Affected repos:** Any repo wanting to display messages (e.g. trace URLs) on shell entry **Symptoms:** @@ -136,11 +140,11 @@ removed as they are no longer needed. devenv's PTY task runner sends two echo sentinels and reads until both are found, feeding all output to a headless VT. This intentionally hides task runner noise but also swallows any user-facing messages from `enterShell`. -**Workaround:** - -Provide an on-demand `otel-trace` shell function instead of auto-displaying. The function is defined during rcfile sourcing and stays available in the interactive shell. +**Current repo approach:** -**Upstream proposal:** A post-drain hook mechanism (env var, file-based, or `ShellCommand` variant) to run code after the interactive session starts. +- Emit OTEL shell-entry notices through `devenv.messages` task output. +- Reuse the exported Grafana link env in `otel-trace` for on-demand reopening. +- Keep a TODO to return to the `v2.0.7` tag once that release is available. --- @@ -243,10 +247,9 @@ Git hooks run in a subprocess that doesn't inherit the direnv environment. - Remove manual JSON trace post-processing from CI pipelines - Update R10 status in this document to reflect full compliance -- **DEVENV-05 fixed (post-drain hook via #2500):** - - Implement auto-display of otel trace URL using the new hook mechanism - - Remove "on-demand only" comment in `nix/devenv-modules/otel.nix` - - Update `context/otel.md` to reflect auto-display capability +- **DEVENV-05 follow-up (tagged release contains #2661):** + - Replace the temporary commit pin with the `v2.0.7` tag + - Remove the temporary pin note from `devenv.yaml` / CI docs - **COMPAT-01 improved (web coding agent support):** - When Claude Code Web adds Nix domains to allowlist: update status, remove "Full internet" workaround diff --git a/devenv.lock b/devenv.lock index 07b4df82c..c472b823b 100644 --- a/devenv.lock +++ b/devenv.lock @@ -162,17 +162,17 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1774168944, - "narHash": "sha256-i1G6n/7Z5fO9RhplzXQSTiLyh1Cs0GhoCoEStFLARtA=", + "lastModified": 1774649847, + "narHash": "sha256-2h7rrOzLjyQdt20yHKPnK0fA+v0fj+whGaDBnmfGahY=", "owner": "cachix", "repo": "devenv", - "rev": "55d2bb4a3cc710ba82cc8644f4419db3a802e1a4", + "rev": "61170924d98492ad8842dca02ad8b912305d308b", "type": "github" }, "original": { "owner": "cachix", - "ref": "v2.0.6", "repo": "devenv", + "rev": "61170924d98492ad8842dca02ad8b912305d308b", "type": "github" } }, diff --git a/devenv.yaml b/devenv.yaml index 162bc5b67..3c3d7c67b 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,6 +1,9 @@ inputs: devenv: - url: github:cachix/devenv/v2.0.6 + # Temporary pin to the post-#2661 commit so shell-entry task messages are + # available before the upstream v2.0.7 release lands. + # TODO: Switch back to github:cachix/devenv/v2.0.7 once it is released. + url: github:cachix/devenv/61170924d98492ad8842dca02ad8b912305d308b nixpkgs: url: github:NixOS/nixpkgs/nixos-unstable git-hooks: diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 74bc4d01b..5f87d5af0 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -622,6 +622,10 @@ export const cachixStep = (opts: { name: string; authToken?: string }) => ({ /** * Prepare lock-pinned devenv metadata from devenv.lock. + * + * The lock may temporarily point at an upstream commit instead of a release tag + * while we validate a fix ahead of the next devenv release. + * TODO: Drop that temporary pin once v2.0.7 is available. */ export const preparePinnedDevenvStep = { name: 'Use pinned devenv from lock', diff --git a/nix/devenv-modules/otel.nix b/nix/devenv-modules/otel.nix index 43c4779a9..53dfc332e 100644 --- a/nix/devenv-modules/otel.nix +++ b/nix/devenv-modules/otel.nix @@ -359,6 +359,74 @@ let # Whether to include local OTEL infrastructure (collector, tempo, grafana processes) needsLocalInfra = mode != "system"; + otelResolveShellState = '' + resolve_otel_shell_state() { + if [ "$OTEL_MODE" = "auto" ]; then + if [ -n "''${OTEL_STATE_DIR:-}" ]; then + OTEL_MODE="system" + else + OTEL_MODE="local" + fi + fi + + if [ "$OTEL_MODE" = "system" ]; then + if [ -z "''${OTEL_STATE_DIR:-}" ]; then + echo "[otel] ERROR: OTEL_MODE=system requires OTEL_STATE_DIR" >&2 + return 1 + fi + if [ -z "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ]; then + echo "[otel] ERROR: OTEL_MODE=system requires OTEL_EXPORTER_OTLP_ENDPOINT" >&2 + return 1 + fi + if [ -z "''${OTEL_GRAFANA_URL:-}" ]; then + echo "[otel] ERROR: OTEL_MODE=system requires OTEL_GRAFANA_URL" >&2 + return 1 + fi + if ! command -v otel >/dev/null 2>&1; then + echo "[otel] ERROR: OTEL_MODE=system requires otel CLI for dashboard sync" >&2 + return 1 + fi + if [ "${toString (builtins.length extraDashboards)}" -gt 0 ]; then + echo "[otel] ERROR: extraDashboards is not supported in OTEL_MODE=system" >&2 + return 1 + fi + if ! otel dash sync \ + --source "${allDashboards}" \ + --target "$OTEL_STATE_DIR/dashboards" >/dev/null 2>&1; then + echo "[otel] ERROR: otel dash sync failed" >&2 + return 1 + fi + _otel_mode_msg="[otel] Using system-level OTEL stack (mode=$OTEL_MODE)" + else + export OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:${toString otelCollectorPort}" + export OTEL_GRAFANA_URL="http://127.0.0.1:${toString grafanaPort}" + export OTEL_SPAN_SPOOL_DIR="${spoolDir}" + _otel_mode_msg="[otel] Using local devenv OTEL stack (mode=$OTEL_MODE)" + fi + + _otel_grafana="$OTEL_GRAFANA_URL" + if [ -n "''${TS_HOSTNAME:-}" ]; then + _otel_grafana="''${_otel_grafana//127.0.0.1/$TS_HOSTNAME}" + fi + if [ -n "''${TRACEPARENT:-}" ]; then + IFS='-' read -r _ _otel_trace_id _ _ <<< "$TRACEPARENT" + _panes='{"a":{"datasource":{"type":"tempo","uid":"tempo"},"queries":[{"refId":"A","datasource":{"type":"tempo","uid":"tempo"},"queryType":"traceql","query":"'"$_otel_trace_id"'"}],"range":{"from":"now-1h","to":"now"}}}' + _encoded=$(printf '%s' "$_panes" | sed 's/{/%7B/g;s/}/%7D/g;s/\[/%5B/g;s/\]/%5D/g;s/"/%22/g;s/:/%3A/g;s/,/%2C/g;s/ /%20/g') + _otel_grafana_link_url="$_otel_grafana/explore?schemaVersion=1&panes=$_encoded&orgId=1" + else + unset _otel_trace_id + _otel_grafana_link_url="$_otel_grafana" + fi + if [ -n "''${_otel_trace_id:-}" ]; then + _otel_trace_label="trace:$_otel_trace_id" + else + _otel_trace_label="grafana" + fi + _otel_grafana_display="$(printf '\e]8;;%s\x07\e[4m%s\e[24m\e]8;;\x07' "$_otel_grafana_link_url" "$_otel_trace_label")" + _otel_start_msg="[otel] Start with: devenv up | $_otel_grafana_display" + } + ''; + in { packages = [ @@ -372,101 +440,35 @@ in env.OTEL_MODE = mode; - # mkAfter ensures this runs after other enterShell code, so env vars - # (including TRACEPARENT from setup:gate) are available. - # Note: devenv's PTY task runner drains all PROMPT_COMMAND output before the - # interactive session, so we provide `otel-trace` for on-demand trace URL access. + # OTEL shell state is resolved in a task so the same source of truth can + # export env vars and emit the post-init shell message via devenv.messages. + # enterShell then only consumes that exported state. enterShell = lib.mkAfter '' - # ── Mode detection ────────────────────────────────────────────────── - # Resolve "auto" to "system" or "local" at runtime. - # Contract: a system-level OTEL stack (e.g. home-manager otel-stack module) - # advertises itself by setting OTEL_STATE_DIR as a session variable. - if [ "$OTEL_MODE" = "auto" ]; then - if [ -n "''${OTEL_STATE_DIR:-}" ]; then - OTEL_MODE="system" - else - OTEL_MODE="local" - fi - fi - - if [ "$OTEL_MODE" = "system" ]; then - if [ -z "''${OTEL_STATE_DIR:-}" ]; then - echo "[otel] ERROR: OTEL_MODE=system requires OTEL_STATE_DIR" >&2 - return 1 2>/dev/null || exit 1 - fi - if [ -z "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ]; then - echo "[otel] ERROR: OTEL_MODE=system requires OTEL_EXPORTER_OTLP_ENDPOINT" >&2 - return 1 2>/dev/null || exit 1 - fi - if ! command -v otel >/dev/null 2>&1; then - echo "[otel] ERROR: OTEL_MODE=system requires otel CLI for dashboard sync" >&2 - return 1 2>/dev/null || exit 1 - fi - if [ "${toString (builtins.length extraDashboards)}" -gt 0 ]; then - echo "[otel] ERROR: extraDashboards is not supported in OTEL_MODE=system" >&2 - return 1 2>/dev/null || exit 1 - fi - if ! otel dash sync \ - --source "${allDashboards}" \ - --target "$OTEL_STATE_DIR/dashboards" >/dev/null 2>&1; then - echo "[otel] ERROR: otel dash sync failed" >&2 - return 1 2>/dev/null || exit 1 - fi - _otel_entry_msg="[otel] Using system-level OTEL stack (mode=$OTEL_MODE)" + # `otel-trace` remains as a cheap on-demand way to reopen the current link, + # but the user-visible shell-entry message now comes from `otel:shell-env`. + otel_trace() { + local _url="''${OTEL_GRAFANA_LINK_URL:-''${OTEL_GRAFANA_URL:-}}" + if [ -z "$_url" ]; then + echo "[otel] No OTEL grafana link available" + return 1 + fi + if [ -n "''${TRACEPARENT:-}" ]; then + IFS='-' read -r _ _tid _ _ <<< "$TRACEPARENT" + local _label="trace:$_tid" + if [ -t 1 ]; then + printf '\e]8;;%s\x07\e[4m%s\e[24m\e]8;;\x07\n' "$_url" "$_label" else - # Local devenv stack — set env vars with local hash-derived ports - export OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:${toString otelCollectorPort}" - export OTEL_GRAFANA_URL="http://127.0.0.1:${toString grafanaPort}" - export OTEL_SPAN_SPOOL_DIR="${spoolDir}" - _otel_entry_msg="[otel] Using local devenv OTEL stack (mode=$OTEL_MODE)" - fi - - _otel_grafana="$OTEL_GRAFANA_URL" - if [ -n "''${TS_HOSTNAME:-}" ]; then - _otel_grafana="''${_otel_grafana//127.0.0.1/$TS_HOSTNAME}" + echo "$_label $_url" fi - # Build Grafana link: trace-specific when TRACEPARENT is available, dashboard otherwise - if [ -n "''${TRACEPARENT:-}" ]; then - IFS='-' read -r _ _otel_trace_id _ _ <<< "$TRACEPARENT" - _panes='{"a":{"datasource":{"type":"tempo","uid":"tempo"},"queries":[{"refId":"A","datasource":{"type":"tempo","uid":"tempo"},"queryType":"traceql","query":"'"$_otel_trace_id"'"}],"range":{"from":"now-1h","to":"now"}}}' - _encoded=$(printf '%s' "$_panes" | sed 's/{/%7B/g;s/}/%7D/g;s/\[/%5B/g;s/\]/%5D/g;s/"/%22/g;s/:/%3A/g;s/,/%2C/g;s/ /%20/g') - _grafana_link_url="$_otel_grafana/explore?schemaVersion=1&panes=$_encoded&orgId=1" + else + if [ -t 1 ]; then + printf '\e]8;;%s\x07\e[4m%s\e[24m\e]8;;\x07\n' "$_url" "grafana" else - _grafana_link_url="$_otel_grafana" + echo "grafana $_url" fi - if [ -n "''${_otel_trace_id:-}" ]; then - _trace_label="trace:$_otel_trace_id" - else - _trace_label="grafana" - fi - if [ -t 2 ]; then - _grafana_display="$(printf '\e]8;;%s\x07\e[4m%s\e[24m\e]8;;\x07' "$_grafana_link_url" "$_trace_label")" - else - _grafana_display="$_trace_label $_grafana_link_url" - fi - _otel_entry_msg="$_otel_entry_msg - [otel] Start with: devenv up | $_grafana_display" - - # devenv's PTY task runner drains all PROMPT_COMMAND output before the - # interactive session starts, so we can't display messages via echo. - # Instead, provide an `otel-trace` shell function for on-demand access. - # No `export -f` needed — function is defined during rcfile sourcing - # and stays available in the interactive shell. - export OTEL_GRAFANA_LINK_URL="$_grafana_link_url" - otel_trace() { - if [ -n "''${TRACEPARENT:-}" ]; then - IFS='-' read -r _ _tid _ _ <<< "$TRACEPARENT" - local _url="''${OTEL_GRAFANA_LINK_URL:-$OTEL_GRAFANA_URL}" - if [ -t 1 ]; then - printf '\e]8;;%s\x07\e[4m%s\e[24m\e]8;;\x07\n' "$_url" "trace:$_tid" - else - echo "trace:$_tid $_url" - fi - else - echo "[otel] No TRACEPARENT available" - fi - } - alias otel-trace=otel_trace + fi + } + alias otel-trace=otel_trace # Detect cold vs warm start (setup-git-hash written by setup.nix) _cold_start="false" @@ -589,6 +591,47 @@ in # Tasks # ========================================================================= + tasks."otel:shell-env" = { + description = "Resolve OTEL shell env and shell-entry message"; + exports = [ + "OTEL_MODE" + "OTEL_EXPORTER_OTLP_ENDPOINT" + "OTEL_GRAFANA_URL" + "OTEL_SPAN_SPOOL_DIR" + "OTEL_GRAFANA_LINK_URL" + ]; + exec = '' + set -euo pipefail + ${otelResolveShellState} + resolve_otel_shell_state + + ${pkgs.jq}/bin/jq -n \ + --arg mode "$OTEL_MODE" \ + --arg endpoint "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" \ + --arg grafanaUrl "''${OTEL_GRAFANA_URL:-}" \ + --arg spoolDir "''${OTEL_SPAN_SPOOL_DIR:-}" \ + --arg linkUrl "$_otel_grafana_link_url" \ + --arg modeMessage "$_otel_mode_msg" \ + --arg startMessage "$_otel_start_msg" \ + '{ + devenv: { + env: ( + { + OTEL_MODE: $mode, + OTEL_GRAFANA_LINK_URL: $linkUrl + } + + (if $endpoint != "" then { OTEL_EXPORTER_OTLP_ENDPOINT: $endpoint } else {} end) + + (if $grafanaUrl != "" then { OTEL_GRAFANA_URL: $grafanaUrl } else {} end) + + (if $spoolDir != "" then { OTEL_SPAN_SPOOL_DIR: $spoolDir } else {} end) + ), + messages: [$modeMessage, $startMessage] + } + }' > "$DEVENV_TASK_OUTPUT_FILE" + ''; + before = [ "devenv:enterShell" ]; + after = lib.optionals (builtins.hasAttr "setup:gate" config.tasks) [ "setup:gate" ]; + }; + tasks."otel:test" = { description = "Run otel-span shell-level unit tests (offline, no devenv up needed)"; exec = '' @@ -607,6 +650,8 @@ in # so always provide a local default for the offline unit tests. export OTEL_EXPORTER_OTLP_ENDPOINT="''${OTEL_EXPORTER_OTLP_ENDPOINT:-http://127.0.0.1:4318}" + ${otelResolveShellState} + _check() { local name="$1" shift @@ -647,7 +692,37 @@ in } _check "Attribute type handling" _test_attr_types - # Test 2: TRACEPARENT propagation + # Test 3: local shell state resolution exports the local stack and a trace link + _test_shell_state_local() { + ( + export OTEL_MODE="local" + export TRACEPARENT="00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01" + export TS_HOSTNAME="ts.example.test" + unset OTEL_GRAFANA_URL OTEL_EXPORTER_OTLP_ENDPOINT OTEL_SPAN_SPOOL_DIR + resolve_otel_shell_state + [ "$OTEL_EXPORTER_OTLP_ENDPOINT" = "http://127.0.0.1:${toString otelCollectorPort}" ] || return 1 + [ "$OTEL_GRAFANA_URL" = "http://127.0.0.1:${toString grafanaPort}" ] || return 1 + [ "$OTEL_SPAN_SPOOL_DIR" = "${spoolDir}" ] || return 1 + echo "$_otel_grafana_link_url" | grep -q 'ts.example.test' || return 1 + echo "$_otel_start_msg" | grep -q 'trace:' || return 1 + ) + } + _check "Shell state resolution (local)" _test_shell_state_local + + # Test 4: system shell state requires an explicit Grafana URL + _test_shell_state_system_requires_grafana() { + ( + export OTEL_MODE="system" + export OTEL_STATE_DIR="$_tmp/system-state" + export OTEL_EXPORTER_OTLP_ENDPOINT="http://collector.example:4318" + unset OTEL_GRAFANA_URL OTEL_SPAN_SPOOL_DIR + otel() { return 0; } + ! resolve_otel_shell_state >/dev/null 2>&1 + ) + } + _check "Shell state resolution (system requires Grafana URL)" _test_shell_state_system_requires_grafana + + # Test 5: TRACEPARENT propagation _test_traceparent() { local spool="$_tmp/tp-test" mkdir -p "$spool" @@ -664,14 +739,14 @@ in } _check "TRACEPARENT propagation" _test_traceparent - # Test 3: Spool fallback (nonexistent dir) + # Test 6: Spool fallback (nonexistent dir) _test_spool_fallback() { # With nonexistent spool dir, should still succeed (falls back to curl which may fail silently) OTEL_SPAN_SPOOL_DIR="/nonexistent" OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:1" otel-span run "test" "fallback" -- true >/dev/null 2>&1 } _check "Spool fallback" _test_spool_fallback - # Test 4: Spool file write + # Test 7: Spool file write _test_spool_write() { local spool="$_tmp/write-test" mkdir -p "$spool" @@ -683,7 +758,7 @@ in } _check "Spool write" _test_spool_write - # Test 5: --span-id override + # Test 8: --span-id override _test_span_id_override() { local spool="$_tmp/spanid-test" mkdir -p "$spool" @@ -695,7 +770,7 @@ in } _check "--span-id override" _test_span_id_override - # Test 6: --start-time-ns override + # Test 9: --start-time-ns override _test_start_time_override() { local spool="$_tmp/startns-test" mkdir -p "$spool" @@ -707,7 +782,7 @@ in } _check "--start-time-ns override" _test_start_time_override - # Test 7: --end-time-ns override + # Test 10: --end-time-ns override _test_end_time_override() { local spool="$_tmp/endns-test" mkdir -p "$spool" @@ -719,7 +794,7 @@ in } _check "--end-time-ns override" _test_end_time_override - # Test 8: --log-url outputs Grafana trace URL to stderr + # Test 11: --log-url outputs Grafana trace URL to stderr _test_log_url() { local spool="$_tmp/logurl-test" mkdir -p "$spool" @@ -736,7 +811,7 @@ in } _check "--log-url output" _test_log_url - # Test 9: No trace context produces root span (no parentSpanId) + # Test 12: No trace context produces root span (no parentSpanId) _test_no_traceparent_root() { local spool="$_tmp/root-test" mkdir -p "$spool" @@ -752,7 +827,7 @@ in } _check "No trace context = root span" _test_no_traceparent_root - # Test 10: OTEL_TASK_TRACEPARENT takes precedence over TRACEPARENT + # Test 13: OTEL_TASK_TRACEPARENT takes precedence over TRACEPARENT _test_task_traceparent_precedence() { local spool="$_tmp/task-tp-test" mkdir -p "$spool" @@ -773,7 +848,7 @@ in } _check "OTEL_TASK_TRACEPARENT precedence" _test_task_traceparent_precedence - # Test 11: --status-attr derives bool from exit code (cached case, exit 0) + # Test 14: --status-attr derives bool from exit code (cached case, exit 0) _test_status_attr_cached() { local spool="$_tmp/status-cached" mkdir -p "$spool" @@ -793,7 +868,7 @@ in } _check "--status-attr cached (exit 0)" _test_status_attr_cached - # Test 12: --status-attr derives bool from exit code (uncached case, exit 1) + # Test 15: --status-attr derives bool from exit code (uncached case, exit 1) _test_status_attr_uncached() { local spool="$_tmp/status-uncached" mkdir -p "$spool" @@ -813,7 +888,7 @@ in } _check "--status-attr uncached (exit 1)" _test_status_attr_uncached - # Test 13: --status-attr propagates TRACEPARENT to child (sub-traces) + # Test 16: --status-attr propagates TRACEPARENT to child (sub-traces) _test_status_attr_subtrace() { local spool="$_tmp/status-subtrace" mkdir -p "$spool" @@ -831,7 +906,7 @@ in } _check "--status-attr sub-trace propagation" _test_status_attr_subtrace - # Test 14: otel-span exports OTEL_TASK_TRACEPARENT to child processes + # Test 17: otel-span exports OTEL_TASK_TRACEPARENT to child processes _test_task_traceparent_export() { local spool="$_tmp/task-tp-export" mkdir -p "$spool" diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 652b596d0..c8b44a388 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -194,14 +194,28 @@ let compute_projection_state_hash() { { # Keep a cheap fingerprint for the realized node_modules projection. - # This catches missing/bad projections on the warm path without the - # deeper link-health scan that made cached installs expensive. + # This catches missing or stale projections on the warm path without + # the deeper dependency-resolution scan that made cached installs + # expensive. Package-level node_modules links are part of the live + # projection contract, so we fingerprint their resolved targets too. for node_modules_dir in node_modules ${nodeModulesPaths}; do if [ -d "$node_modules_dir" ]; then printf 'dir %s\n' "$node_modules_dir" else printf 'missing %s\n' "$node_modules_dir" + continue fi + + find "$node_modules_dir" -mindepth 1 -maxdepth 2 -type l -print \ + | LC_ALL=C sort \ + | while IFS= read -r link_path; do + link_target="$(readlink "$link_path" || true)" + if [ -e "$link_path" ]; then + printf 'link %s -> %s\n' "$link_path" "$link_target" + else + printf 'broken-link %s -> %s\n' "$link_path" "$link_target" + fi + done done if [ -f node_modules/.modules.yaml ]; then diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index ffca45b21..b3f569824 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -316,7 +316,24 @@ echo "Test 5: exec restores projection metadata after a miss" test -f "$workspace/node_modules/.modules.yaml" ) -echo "Test 6: exec defaults PNPM_HOME to a workspace-local projection" +echo "Test 6: outer cache hit misses when a projected package symlink breaks" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + export DEVENV_SETUP_OUTER_CACHE_HIT=1 + mkdir -p node_modules/@scope + ln -s ../missing-package node_modules/@scope/broken + set +e + bash "$tmpdir/pnpm-install.status.sh" + exit_code=$? + set -e + unset DEVENV_SETUP_OUTER_CACHE_HIT + assert_exit_code 1 "$exit_code" "outer-hit status should miss when a projected symlink is broken" + rm node_modules/@scope/broken +) + +echo "Test 7: exec defaults PNPM_HOME to a workspace-local projection" ( cd "$workspace" export HOME="$tmpdir/home" @@ -328,7 +345,7 @@ echo "Test 6: exec defaults PNPM_HOME to a workspace-local projection" grep -qxF "npm_config_store_dir=$workspace/.direnv/pnpm-store" "$tmpdir/pnpm.log" ) -echo "Test 7: status hits after install with the default GVS path" +echo "Test 8: status hits after install with the default GVS path" ( cd "$workspace" export HOME="$tmpdir/home" @@ -340,7 +357,7 @@ echo "Test 7: status hits after install with the default GVS path" assert_exit_code 0 "$exit_code" "status should hit after default-PNPM_HOME install" ) -echo "Test 8: status still hits when PNPM_HOME changes but store-dir stays shared" +echo "Test 9: status still hits when PNPM_HOME changes but store-dir stays shared" ( cd "$workspace" export HOME="$tmpdir/home" @@ -352,7 +369,7 @@ echo "Test 8: status still hits when PNPM_HOME changes but store-dir stays share assert_exit_code 0 "$exit_code" "status should hit when only PNPM_HOME changes" ) -echo "Test 9: status misses after effective store-dir changes" +echo "Test 10: status misses after effective store-dir changes" ( cd "$workspace" export HOME="$tmpdir/home" @@ -365,10 +382,10 @@ echo "Test 9: status misses after effective store-dir changes" assert_exit_code 1 "$exit_code" "status should miss when store-dir changes" ) -echo "Test 10: exec invoked pnpm install" +echo "Test 11: exec invoked pnpm install" grep -q "^install " "$tmpdir/pnpm.log" -echo "Test 11: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" +echo "Test 12: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" ( cd "$workspace" export HOME="$tmpdir/home" @@ -385,7 +402,7 @@ echo "Test 11: nested workspace exec uses its own cwd, cache, PNPM_HOME, and sha grep -qxF "npm_config_store_dir=$workspace/.direnv/pnpm-store" "$tmpdir/pnpm.log" ) -echo "Test 12: nested workspace status hits after nested install" +echo "Test 13: nested workspace status hits after nested install" ( cd "$workspace" export HOME="$tmpdir/home" @@ -399,7 +416,7 @@ echo "Test 12: nested workspace status hits after nested install" assert_exit_code 0 "$exit_code" "nested status should hit after nested install" ) -echo "Test 13: install flags and pre-install hooks are applied" +echo "Test 14: install flags and pre-install hooks are applied" ( cd "$workspace" export HOME="$tmpdir/home" @@ -413,7 +430,7 @@ echo "Test 13: install flags and pre-install hooks are applied" grep -qxF "install --config.confirmModulesPurge=false --config.store-dir=$workspace/.direnv/pnpm-store --ignore-scripts --config.public-hoist-pattern=*" "$tmpdir/pnpm.log" ) -echo "Test 14: CI install failures preserve and classify the pnpm log" +echo "Test 15: CI install failures preserve and classify the pnpm log" ( cd "$workspace" export HOME="$tmpdir/home" @@ -437,21 +454,21 @@ echo "Test 14: CI install failures preserve and classify the pnpm log" grep -qF "Socket timeout" <<< "$output" ) -echo "Test 15: generated test task runs vitest without pnpm exec" +echo "Test 16: generated test task runs vitest without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/test-demo.exec.sh")" [ "$output" = "vitest-shim:run" ] ) -echo "Test 16: generated storybook task runs storybook without pnpm exec" +echo "Test 17: generated storybook task runs storybook without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/storybook-demo.exec.sh")" [ "$output" = "storybook-shim:build" ] ) -echo "Test 17: clean leaves shared GVS links intact" +echo "Test 18: clean leaves shared GVS links intact" ( cd "$workspace" mkdir -p "$workspace/.direnv/pnpm-store/v11/links/shared-pkg" From 436cfa9910c94b1cef356c07d3c9c3d7009fad50 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Sat, 28 Mar 2026 11:10:39 +0100 Subject: [PATCH 09/14] Fix OTEL shell entry root tracing --- CHANGELOG.md | 2 + nix/devenv-modules/otel.nix | 295 ++++++++++++++++++++++++++++-------- 2 files changed, 231 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5896606c3..e9ef632da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,8 @@ All notable changes to this project will be documented in this file. - Resolves OTEL mode, dashboard sync, and Grafana trace-link construction in a dedicated shell-entry task instead of ad-hoc `enterShell` output - Auto-displays the OTEL shell-entry message through upstream task messages while keeping `otel-trace` as a lightweight re-open helper - Adds a TODO to switch the temporary commit pin back to the `v2.0.7` tag once it is released + - Scrubs ambient task trace context before emitting `devenv/shell:entry` so the shell root span cannot self-parent or collide with later `dt` root spans + - Emits `devenv/shell:entry` via the pinned store path for `otel-span` so tracing still works before `enterShell` PATH mutations are fully visible - **@overeng/genie**: Validate GitHub Actions `runs-on` labels before emitting workflow YAML - Fails `genie` when workflow jobs serialize non-string, empty, or stale placeholder runner labels like `null` / `...=undefined` - Prevents CI helper API drift from silently generating invalid workflow files that only fail later in GitHub Actions diff --git a/nix/devenv-modules/otel.nix b/nix/devenv-modules/otel.nix index 53dfc332e..0fe7c0bca 100644 --- a/nix/devenv-modules/otel.nix +++ b/nix/devenv-modules/otel.nix @@ -427,6 +427,88 @@ let } ''; + otelDetectShellEntryState = '' + detect_otel_shell_entry_state() { + # Detect cold vs warm start (setup-git-hash written by setup.nix) + _cold_start="false" + if [ ! -f .direnv/task-cache/setup-git-hash ]; then + _cold_start="true" + elif [ "$(git rev-parse HEAD 2>/dev/null || echo no-git)" != "$(cat .direnv/task-cache/setup-git-hash 2>/dev/null || echo "")" ]; then + _cold_start="true" + fi + + # Detect what triggered this shell reload by comparing watched file mtimes. + # Uses devenv's input-paths.txt (nix inputs that affect the shell derivation), + # excluding .devenv/bootstrap/ files which are regenerated on every eval. + # Missing paths are tolerated here because input files can legitimately + # disappear between eval and shell startup while the user is editing. + _reload_trigger="unknown" + _otel_mtime_snapshot=".direnv/otel-watch-mtimes" + if [ -f ".devenv/input-paths.txt" ]; then + _otel_current=$( + while IFS= read -r _otel_path; do + [ -n "$_otel_path" ] || continue + [ -e "$_otel_path" ] || continue + ${pkgs.coreutils}/bin/stat -c '%Y %n' "$_otel_path" + done < <(${pkgs.gnugrep}/bin/grep -v '\.devenv/bootstrap/' .devenv/input-paths.txt) \ + | ${pkgs.coreutils}/bin/sort -k2 + ) + if [ ! -f "$_otel_mtime_snapshot" ]; then + _reload_trigger="initial" + elif [ "$_otel_current" = "$(${pkgs.coreutils}/bin/cat "$_otel_mtime_snapshot" 2>/dev/null)" ]; then + _reload_trigger="env-change" + else + _otel_changed=$( + (${pkgs.diffutils}/bin/diff <(${pkgs.coreutils}/bin/cat "$_otel_mtime_snapshot") <(echo "$_otel_current") 2>/dev/null || true) \ + | ${pkgs.gnugrep}/bin/grep '^[<>]' | ${pkgs.gawk}/bin/awk '{print $NF}' | ${pkgs.coreutils}/bin/sort -u \ + | ${pkgs.gnused}/bin/sed "s|^''${DEVENV_ROOT:-.}/||" \ + | ${pkgs.coreutils}/bin/head -5 | ${pkgs.coreutils}/bin/paste -sd ',' - + ) + _reload_trigger="''${_otel_changed:-unknown}" + fi + ${pkgs.coreutils}/bin/mkdir -p .direnv + echo "$_otel_current" > "$_otel_mtime_snapshot" + fi + } + ''; + + otelEmitShellEntry = '' + emit_otel_shell_entry_span() { + if [ -z "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ] \ + || [ -z "''${TRACEPARENT:-}" ] \ + || [ -z "''${OTEL_SHELL_ENTRY_NS:-}" ]; then + return 0 + fi + + IFS='-' read -r _ _otel_shell_trace_id _otel_shell_root_span_id _ <<< "$TRACEPARENT" + + # Shell-root tracing must use the store path directly instead of relying + # on PATH, because both shell hooks and early shell-entry tasks can run + # before package PATH mutations are fully visible. + _otel_span_bin="${otelSpan}/bin/otel-span" + [ -x "$_otel_span_bin" ] || return 0 + + # enterShell can run after traced setup tasks. If we let otel-span infer a + # parent from the ambient TRACEPARENT/OTEL_TASK_TRACEPARENT here, the + # shell root span can become self-parented or collide with later root + # spans. Emit it from explicit shell IDs instead. + ( + unset TRACEPARENT OTEL_TASK_TRACEPARENT + "$_otel_span_bin" run "devenv" "shell:entry" \ + --trace-id "$_otel_shell_trace_id" \ + --span-id "$_otel_shell_root_span_id" \ + --start-time-ns "$OTEL_SHELL_ENTRY_NS" \ + --end-time-ns "$(${pkgs.coreutils}/bin/date +%s%N)" \ + --attr "cold_start=$_cold_start" \ + --attr "reload.trigger=$_reload_trigger" \ + -- true + ) || true + + export TRACEPARENT="00-$_otel_shell_trace_id-$_otel_shell_root_span_id-01" + unset OTEL_TASK_TRACEPARENT OTEL_SHELL_ENTRY_NS + } + ''; + in { packages = [ @@ -442,7 +524,8 @@ in # OTEL shell state is resolved in a task so the same source of truth can # export env vars and emit the post-init shell message via devenv.messages. - # enterShell then only consumes that exported state. + # The shell root span is emitted in a dedicated task after setup work, so + # enterShell only consumes exported state and marks the interactive handoff. enterShell = lib.mkAfter '' # `otel-trace` remains as a cheap on-demand way to reopen the current link, # but the user-visible shell-entry message now comes from `otel:shell-env`. @@ -470,58 +553,10 @@ in } alias otel-trace=otel_trace - # Detect cold vs warm start (setup-git-hash written by setup.nix) - _cold_start="false" - if [ ! -f .direnv/task-cache/setup-git-hash ]; then - _cold_start="true" - elif [ "$(git rev-parse HEAD 2>/dev/null || echo no-git)" != "$(cat .direnv/task-cache/setup-git-hash 2>/dev/null || echo "")" ]; then - _cold_start="true" - fi - - # Detect what triggered this shell reload by comparing watched file mtimes. - # Uses devenv's input-paths.txt (nix inputs that affect the shell derivation), - # excluding .devenv/bootstrap/ files which are regenerated on every eval. - # xargs stat is ~2ms for ~50 files — negligible overhead. - _reload_trigger="unknown" - _otel_mtime_snapshot=".direnv/otel-watch-mtimes" - if [ -f ".devenv/input-paths.txt" ]; then - _otel_current=$(grep -v '\.devenv/bootstrap/' .devenv/input-paths.txt \ - | xargs stat -c '%Y %n' 2>/dev/null | sort -k2) - if [ ! -f "$_otel_mtime_snapshot" ]; then - _reload_trigger="initial" - elif [ "$_otel_current" = "$(cat "$_otel_mtime_snapshot" 2>/dev/null)" ]; then - _reload_trigger="env-change" - else - _otel_changed=$(diff <(cat "$_otel_mtime_snapshot") <(echo "$_otel_current") 2>/dev/null \ - | grep '^[<>]' | awk '{print $NF}' | sort -u \ - | sed "s|^''${DEVENV_ROOT:-.}/||" \ - | head -5 | paste -sd ',' -) - _reload_trigger="''${_otel_changed:-unknown}" - fi - mkdir -p .direnv - echo "$_otel_current" > "$_otel_mtime_snapshot" - fi - - # Emit root shell:entry span covering the full setup duration. - # TRACEPARENT and OTEL_SHELL_ENTRY_NS are propagated from setup:gate via - # devenv's native task output -> env mechanism (devenv.env convention). - if command -v otel-span >/dev/null 2>&1 \ - && [ -n "''${OTEL_EXPORTER_OTLP_ENDPOINT:-}" ] \ - && [ -n "''${TRACEPARENT:-}" ] \ - && [ -n "''${OTEL_SHELL_ENTRY_NS:-}" ]; then - IFS='-' read -r _ _trace_id _span_id _ <<< "$TRACEPARENT" - ( - unset TRACEPARENT - otel-span run "devenv" "shell:entry" \ - --trace-id "$_trace_id" \ - --span-id "$_span_id" \ - --start-time-ns "$OTEL_SHELL_ENTRY_NS" \ - --end-time-ns "$(date +%s%N)" \ - --attr "cold_start=$_cold_start" \ - --attr "reload.trigger=$_reload_trigger" \ - -- true - ) || true - fi + # setup:gate seeds shell-root trace IDs for setup tasks. Clear the + # task-scoped context markers before handing control to the interactive + # shell so later `dt` roots do not accidentally reuse shell bootstrap state. + unset OTEL_TASK_TRACEPARENT OTEL_SHELL_ENTRY_NS # Mark the moment the shell becomes interactive (after all setup + OTEL work). # Consumed by dt.nix for the shell.ready_ms span attribute. @@ -632,6 +667,27 @@ in after = lib.optionals (builtins.hasAttr "setup:gate" config.tasks) [ "setup:gate" ]; }; + tasks."otel:shell-entry" = { + description = "Emit the shell-entry root trace span after setup completes"; + exec = '' + set -euo pipefail + ${otelDetectShellEntryState} + ${otelEmitShellEntry} + detect_otel_shell_entry_state || true + emit_otel_shell_entry_span + ''; + before = [ "devenv:enterShell" ]; + after = + lib.optionals (builtins.hasAttr "devenv:files:cleanup" config.tasks) [ "devenv:files:cleanup" ] + ++ lib.optionals (builtins.hasAttr "devenv:files" config.tasks) [ "devenv:files" ] + ++ [ "otel:shell-env" ] + ++ lib.optionals (builtins.hasAttr "setup:record-cache" config.tasks) [ "setup:record-cache@completed" ] + ++ lib.optionals ( + !(builtins.hasAttr "setup:record-cache" config.tasks) + && builtins.hasAttr "setup:gate" config.tasks + ) [ "setup:gate" ]; + }; + tasks."otel:test" = { description = "Run otel-span shell-level unit tests (offline, no devenv up needed)"; exec = '' @@ -651,6 +707,8 @@ in export OTEL_EXPORTER_OTLP_ENDPOINT="''${OTEL_EXPORTER_OTLP_ENDPOINT:-http://127.0.0.1:4318}" ${otelResolveShellState} + ${otelDetectShellEntryState} + ${otelEmitShellEntry} _check() { local name="$1" @@ -722,7 +780,112 @@ in } _check "Shell state resolution (system requires Grafana URL)" _test_shell_state_system_requires_grafana - # Test 5: TRACEPARENT propagation + # Test 5: shell:entry emission uses explicit shell IDs and ignores ambient parents + _test_shell_entry_root_span() { + local spool="$_tmp/shell-entry-root" + mkdir -p "$spool" + ( + export OTEL_SPAN_SPOOL_DIR="$spool" + export TRACEPARENT="00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01" + export OTEL_SHELL_ENTRY_NS="1234567890000000000" + export OTEL_TASK_TRACEPARENT="00-feedfacefeedfacefeedfacefeedface-2222222222222222-01" + _cold_start="false" + _reload_trigger="initial" + + emit_otel_shell_entry_span + + [ "$TRACEPARENT" = "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01" ] || return 1 + [ -z "''${OTEL_TASK_TRACEPARENT:-}" ] || return 1 + [ -z "''${OTEL_SHELL_ENTRY_NS:-}" ] || return 1 + ) + + [ -f "$spool/spans.jsonl" ] || return 1 + + local line actual_trace actual_span has_parent + line=$(head -1 "$spool/spans.jsonl") + actual_trace=$(echo "$line" | ${pkgs.jq}/bin/jq -r '.resourceSpans[0].scopeSpans[0].spans[0].traceId') + actual_span=$(echo "$line" | ${pkgs.jq}/bin/jq -r '.resourceSpans[0].scopeSpans[0].spans[0].spanId') + has_parent=$(echo "$line" | ${pkgs.jq}/bin/jq -r '.resourceSpans[0].scopeSpans[0].spans[0] | has("parentSpanId")') + + [ "$actual_trace" = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ] \ + && [ "$actual_span" = "bbbbbbbbbbbbbbbb" ] \ + && [ "$has_parent" = "false" ] + } + _check "shell:entry root span emission" _test_shell_entry_root_span + + # Test 6: shell:entry emission must not depend on PATH already containing + # otel-span because enterShell can run before package PATH setup settles. + _test_shell_entry_root_span_without_path() { + local spool="$_tmp/shell-entry-no-path" + mkdir -p "$spool" + ( + export OTEL_SPAN_SPOOL_DIR="$spool" + export OTEL_EXPORTER_OTLP_ENDPOINT="http://collector.example:4318" + export TRACEPARENT="00-cccccccccccccccccccccccccccccccc-dddddddddddddddd-01" + export OTEL_SHELL_ENTRY_NS="1234567890000000001" + export PATH="/nonexistent" + _cold_start="false" + _reload_trigger="env-change" + + emit_otel_shell_entry_span + + [ "$TRACEPARENT" = "00-cccccccccccccccccccccccccccccccc-dddddddddddddddd-01" ] || return 1 + [ -z "''${OTEL_SHELL_ENTRY_NS:-}" ] || return 1 + ) + + [ -f "$spool/spans.jsonl" ] || return 1 + + local line actual_trace actual_span has_parent + line=$(head -1 "$spool/spans.jsonl") + actual_trace=$(echo "$line" | ${pkgs.jq}/bin/jq -r '.resourceSpans[0].scopeSpans[0].spans[0].traceId') + actual_span=$(echo "$line" | ${pkgs.jq}/bin/jq -r '.resourceSpans[0].scopeSpans[0].spans[0].spanId') + has_parent=$(echo "$line" | ${pkgs.jq}/bin/jq -r '.resourceSpans[0].scopeSpans[0].spans[0] | has("parentSpanId")') + + [ "$actual_trace" = "cccccccccccccccccccccccccccccccc" ] \ + && [ "$actual_span" = "dddddddddddddddd" ] \ + && [ "$has_parent" = "false" ] + } + _check "shell:entry root span emission without PATH" _test_shell_entry_root_span_without_path + + # Test 7: reload-trigger detection uses pinned binaries instead of + # ambient PATH, so the shell-entry task works before GNU tools are added. + _test_shell_entry_state_without_path() { + local workdir="$_tmp/shell-entry-state-no-path" + mkdir -p "$workdir/.devenv" "$workdir/.direnv" + echo "$workdir/foo.nix" > "$workdir/.devenv/input-paths.txt" + echo "x = 1;" > "$workdir/foo.nix" + + ( + cd "$workdir" + export PATH="/nonexistent" + detect_otel_shell_entry_state + [ "$_cold_start" = "true" ] || return 1 + [ "$_reload_trigger" = "initial" ] || return 1 + [ -f ".direnv/otel-watch-mtimes" ] || return 1 + ) + } + _check "shell-entry state detection without PATH" _test_shell_entry_state_without_path + + # Test 8: reload-trigger detection tolerates input paths that disappear + # between eval and shell startup instead of failing the shell-entry task. + _test_shell_entry_state_missing_paths() { + local workdir="$_tmp/shell-entry-state-missing-paths" + mkdir -p "$workdir/.devenv" "$workdir/.direnv" + echo "$workdir/foo.nix" > "$workdir/.devenv/input-paths.txt" + echo "$workdir/missing.nix" >> "$workdir/.devenv/input-paths.txt" + echo "x = 1;" > "$workdir/foo.nix" + + ( + cd "$workdir" + export PATH="/nonexistent" + detect_otel_shell_entry_state + [ "$_reload_trigger" = "initial" ] || return 1 + [ -f ".direnv/otel-watch-mtimes" ] || return 1 + ) + } + _check "shell-entry state detection with missing paths" _test_shell_entry_state_missing_paths + + # Test 9: TRACEPARENT propagation _test_traceparent() { local spool="$_tmp/tp-test" mkdir -p "$spool" @@ -739,14 +902,14 @@ in } _check "TRACEPARENT propagation" _test_traceparent - # Test 6: Spool fallback (nonexistent dir) + # Test 10: Spool fallback (nonexistent dir) _test_spool_fallback() { # With nonexistent spool dir, should still succeed (falls back to curl which may fail silently) OTEL_SPAN_SPOOL_DIR="/nonexistent" OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:1" otel-span run "test" "fallback" -- true >/dev/null 2>&1 } _check "Spool fallback" _test_spool_fallback - # Test 7: Spool file write + # Test 11: Spool file write _test_spool_write() { local spool="$_tmp/write-test" mkdir -p "$spool" @@ -758,7 +921,7 @@ in } _check "Spool write" _test_spool_write - # Test 8: --span-id override + # Test 9: --span-id override _test_span_id_override() { local spool="$_tmp/spanid-test" mkdir -p "$spool" @@ -770,7 +933,7 @@ in } _check "--span-id override" _test_span_id_override - # Test 9: --start-time-ns override + # Test 10: --start-time-ns override _test_start_time_override() { local spool="$_tmp/startns-test" mkdir -p "$spool" @@ -782,7 +945,7 @@ in } _check "--start-time-ns override" _test_start_time_override - # Test 10: --end-time-ns override + # Test 11: --end-time-ns override _test_end_time_override() { local spool="$_tmp/endns-test" mkdir -p "$spool" @@ -794,7 +957,7 @@ in } _check "--end-time-ns override" _test_end_time_override - # Test 11: --log-url outputs Grafana trace URL to stderr + # Test 12: --log-url outputs Grafana trace URL to stderr _test_log_url() { local spool="$_tmp/logurl-test" mkdir -p "$spool" @@ -811,7 +974,7 @@ in } _check "--log-url output" _test_log_url - # Test 12: No trace context produces root span (no parentSpanId) + # Test 13: No trace context produces root span (no parentSpanId) _test_no_traceparent_root() { local spool="$_tmp/root-test" mkdir -p "$spool" @@ -827,7 +990,7 @@ in } _check "No trace context = root span" _test_no_traceparent_root - # Test 13: OTEL_TASK_TRACEPARENT takes precedence over TRACEPARENT + # Test 14: OTEL_TASK_TRACEPARENT takes precedence over TRACEPARENT _test_task_traceparent_precedence() { local spool="$_tmp/task-tp-test" mkdir -p "$spool" @@ -848,7 +1011,7 @@ in } _check "OTEL_TASK_TRACEPARENT precedence" _test_task_traceparent_precedence - # Test 14: --status-attr derives bool from exit code (cached case, exit 0) + # Test 15: --status-attr derives bool from exit code (cached case, exit 0) _test_status_attr_cached() { local spool="$_tmp/status-cached" mkdir -p "$spool" @@ -868,7 +1031,7 @@ in } _check "--status-attr cached (exit 0)" _test_status_attr_cached - # Test 15: --status-attr derives bool from exit code (uncached case, exit 1) + # Test 16: --status-attr derives bool from exit code (uncached case, exit 1) _test_status_attr_uncached() { local spool="$_tmp/status-uncached" mkdir -p "$spool" @@ -888,7 +1051,7 @@ in } _check "--status-attr uncached (exit 1)" _test_status_attr_uncached - # Test 16: --status-attr propagates TRACEPARENT to child (sub-traces) + # Test 17: --status-attr propagates TRACEPARENT to child (sub-traces) _test_status_attr_subtrace() { local spool="$_tmp/status-subtrace" mkdir -p "$spool" @@ -906,7 +1069,7 @@ in } _check "--status-attr sub-trace propagation" _test_status_attr_subtrace - # Test 17: otel-span exports OTEL_TASK_TRACEPARENT to child processes + # Test 18: otel-span exports OTEL_TASK_TRACEPARENT to child processes _test_task_traceparent_export() { local spool="$_tmp/task-tp-export" mkdir -p "$spool" From 330c78206dbe0b6985d792d7909dc6999154878d Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Sat, 28 Mar 2026 13:23:09 +0100 Subject: [PATCH 10/14] Optimize pnpm projection status hashing --- CHANGELOG.md | 2 +- .../check-node-modules-projection-health.cjs | 196 ++++++++++++------ nix/devenv-modules/tasks/shared/pnpm.nix | 46 +--- .../shared/tests/pnpm-task-smoke.test.sh | 40 +++- 4 files changed, 170 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9ef632da..88ba5eac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,7 +97,7 @@ All notable changes to this project will be documented in this file. - **devenv/tasks**: make warm shell bootstrap commit-scoped and remove `ts:emit` from shell entry - Adds an outer `setup:auto` cache so warm `devenv shell` skips unchanged bootstrap work instead of traversing `pnpm:install`, `genie:run`, and `mr:apply` on every entry - Switches shell bootstrap from `mr:sync` to initial `mr:apply` so a fresh worktree is normalized without fetching on every shell - - Speeds up warm task status paths by using direct `mr status`, fingerprint-based `genie:run` caching, lighter `pnpm:install` status checks, and a `ts:emit` graph that excludes `noEmit` references at emit time + - Speeds up warm task status paths by using direct `mr status`, fingerprint-based `genie:run` caching, a one-process `pnpm:install` projection hash that preserves the previous structural guarantees, and a `ts:emit` graph that excludes `noEmit` references at emit time - Hardens the fast paths by making the outer cache only track setup inputs while each task still verifies its own outputs before skipping - **devenv/otel**: pin `devenv` temporarily to the post-#2661 upstream commit and move OTEL shell-entry notices onto `devenv.messages` - Resolves OTEL mode, dashboard sync, and Grafana trace-link construction in a dedicated shell-entry task instead of ad-hoc `enterShell` output diff --git a/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs b/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs index c5e8b801e..cd638d23a 100644 --- a/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs +++ b/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs @@ -1,24 +1,35 @@ const fs = require('fs') const path = require('path') const { createRequire } = require('module') +const crypto = require('crypto') + +const mode = process.env.NODE_MODULES_HELPER_MODE || 'health' -/** - * GVS can leave package symlinks present while still dropping transitive - * projections after config/path changes. Checking only for broken symlinks - * misses that failure mode, so this helper resolves each symlinked package's - * declared runtime deps from the package's real path. - */ const moduleDirs = (process.env.NODE_MODULES_DIRS || '') .split('\n') .map((value) => value.trim()) .filter(Boolean) .filter((value, index, values) => values.indexOf(value) === index) - .filter((value) => fs.existsSync(value)) -const dependencyProjectionFailures = [] -const packageContentFailures = [] +const existingModuleDirs = moduleDirs.filter((value) => fs.existsSync(value)) -const collectEntryPaths = (nodeModulesDir) => { +const collectProjectionEntryPaths = (nodeModulesDir) => { + const result = [] + for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) { + const entryPath = path.join(nodeModulesDir, entry.name) + if (entry.isDirectory()) { + for (const childEntry of fs.readdirSync(entryPath, { withFileTypes: true })) { + result.push(path.join(entryPath, childEntry.name)) + } + continue + } + + result.push(entryPath) + } + return result.sort() +} + +const collectHealthEntryPaths = (nodeModulesDir) => { const result = [] for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) { if (entry.name === '.bin' || entry.name === '.pnpm') continue @@ -36,12 +47,6 @@ const collectEntryPaths = (nodeModulesDir) => { return result } -/** - * `require.resolve("${dependencyName}/package.json")` is not a valid health - * check because many packages intentionally do not export that subpath. We - * need to verify the projected package directory itself is reachable via Node's - * package search paths, independent of its public exports surface. - */ const resolveDependencyPackageRoot = ({ requireFromPkg, dependencyName }) => { const packagePath = dependencyName.split('/') const searchPaths = requireFromPkg.resolve.paths(dependencyName) ?? [] @@ -59,16 +64,6 @@ const resolveDependencyPackageRoot = ({ requireFromPkg, dependencyName }) => { const isDeclarationTarget = (value) => value.endsWith('.d.ts') || value.endsWith('.d.mts') || value.endsWith('.d.cts') -/** - * Only runtime export targets prove whether the package projection can be - * loaded. Declaration-only branches are intentionally ignored: several packages - * publish type conditions that are absent from the GVS link projection while - * their runtime `default` / `import` targets are present and load correctly. - * See https://github.com/pnpm/pnpm/issues/11385 for the stale runtime-export - * projection scenario this check guards. - * TODO(pnpm#11385): remove this package-content check if pnpm starts repairing - * incomplete GVS link projections during forced installs. - */ const collectRuntimeExportTargets = (value, conditionName = undefined) => { if (typeof value === 'string') { if (conditionName === 'types' || isDeclarationTarget(value)) return [] @@ -88,7 +83,7 @@ const collectRuntimeExportTargets = (value, conditionName = undefined) => { return [] } -const verifyPackageContent = ({ pkg, packageDir, entryPath }) => { +const verifyPackageContent = ({ pkg, packageDir, entryPath, failures }) => { if (!packageDir.includes('/v11/links/')) return const includedFiles = Array.isArray(pkg.files) @@ -121,64 +116,133 @@ const verifyPackageContent = ({ pkg, packageDir, entryPath }) => { const resolved = path.resolve(packageDir, target) if (!fs.existsSync(resolved)) { - packageContentFailures.push(`${pkg.name ?? entryPath} -> ${target} (${packageDir})`) + failures.push(`${pkg.name ?? entryPath} -> ${target} (${packageDir})`) } } } -for (const nodeModulesDir of moduleDirs) { - for (const entryPath of collectEntryPaths(nodeModulesDir)) { - let stat - try { - stat = fs.lstatSync(entryPath) - } catch { +const runProjectionHash = () => { + const hash = crypto.createHash('sha256') + const appendLine = (line) => { + hash.update(line) + hash.update('\n') + } + + for (const nodeModulesDir of moduleDirs) { + if (fs.existsSync(nodeModulesDir) && fs.statSync(nodeModulesDir).isDirectory()) { + appendLine(`dir ${nodeModulesDir}`) + } else { + appendLine(`missing ${nodeModulesDir}`) continue } - if (!stat.isSymbolicLink()) continue + for (const entryPath of collectProjectionEntryPaths(nodeModulesDir)) { + let stat + try { + stat = fs.lstatSync(entryPath) + } catch { + continue + } - let realPath - try { - realPath = fs.realpathSync(entryPath) - } catch { - continue - } + if (!stat.isSymbolicLink()) continue - const packageJsonPath = path.join(realPath, 'package.json') - if (!fs.existsSync(packageJsonPath)) continue - - const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) - verifyPackageContent({ pkg, packageDir: realPath, entryPath }) - - const dependencyNames = Object.keys(pkg.dependencies ?? {}) - if (dependencyNames.length === 0) continue - - const requireFromPkg = createRequire(packageJsonPath) - for (const dependencyName of dependencyNames) { - if ( - resolveDependencyPackageRoot({ - requireFromPkg, - dependencyName, - }) === undefined - ) { - dependencyProjectionFailures.push( - `${pkg.name ?? entryPath} -> ${dependencyName} (from ${nodeModulesDir})`, - ) + let target = '' + try { + target = fs.readlinkSync(entryPath) + } catch {} + + if (fs.existsSync(entryPath)) { + appendLine(`link ${entryPath} -> ${target}`) + } else { + appendLine(`broken-link ${entryPath} -> ${target}`) } } } + + const rootModulesYamlPath = process.env.PNPM_ROOT_MODULES_YAML || 'node_modules/.modules.yaml' + if (fs.existsSync(rootModulesYamlPath)) { + appendLine( + `modules-yaml ${crypto + .createHash('sha256') + .update(fs.readFileSync(rootModulesYamlPath)) + .digest('hex')}`, + ) + } else { + appendLine('modules-yaml missing') + } + + process.stdout.write(`${hash.digest('hex')}\n`) } -if (dependencyProjectionFailures.length > 0) { +const runHealthCheck = () => { + const dependencyProjectionFailures = [] + const packageContentFailures = [] + + for (const nodeModulesDir of existingModuleDirs) { + for (const entryPath of collectHealthEntryPaths(nodeModulesDir)) { + let stat + try { + stat = fs.lstatSync(entryPath) + } catch { + continue + } + + if (!stat.isSymbolicLink()) continue + + let realPath + try { + realPath = fs.realpathSync(entryPath) + } catch { + continue + } + + const packageJsonPath = path.join(realPath, 'package.json') + if (!fs.existsSync(packageJsonPath)) continue + + const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) + verifyPackageContent({ + pkg, + packageDir: realPath, + entryPath, + failures: packageContentFailures, + }) + + const dependencyNames = Object.keys(pkg.dependencies ?? {}) + if (dependencyNames.length === 0) continue + + const requireFromPkg = createRequire(packageJsonPath) + for (const dependencyName of dependencyNames) { + if ( + resolveDependencyPackageRoot({ + requireFromPkg, + dependencyName, + }) === undefined + ) { + dependencyProjectionFailures.push( + `${pkg.name ?? entryPath} -> ${dependencyName} (from ${nodeModulesDir})`, + ) + } + } + } + } + for (const failure of dependencyProjectionFailures) { console.error(`[pnpm] Missing dependency projection: ${failure}`) } - process.exit(1) -} - -if (packageContentFailures.length > 0) { for (const failure of packageContentFailures) { console.error(`[pnpm] Missing package content: ${failure}`) } + + if (dependencyProjectionFailures.length > 0 || packageContentFailures.length > 0) { + process.exit(1) + } +} + +if (mode === 'projection-hash') { + runProjectionHash() +} else if (mode === 'health') { + runHealthCheck() +} else { + console.error(`[pnpm] Unknown node_modules helper mode: ${mode}`) process.exit(1) } diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index c8b44a388..54ef8e128 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -68,7 +68,7 @@ let pnpmTaskHelpersScript = pkgs.writeText "pnpm-task-helpers.sh" ( builtins.readFile ./pnpm-task-helpers.sh ); - nodeModulesProjectionHealthScript = pkgs.writeText "check-node-modules-projection-health.cjs" ( + nodeModulesProjectionScript = pkgs.writeText "check-node-modules-projection-health.cjs" ( builtins.readFile ./check-node-modules-projection-health.cjs ); @@ -192,39 +192,13 @@ let ''; computeProjectionStateHashFn = '' compute_projection_state_hash() { - { - # Keep a cheap fingerprint for the realized node_modules projection. - # This catches missing or stale projections on the warm path without - # the deeper dependency-resolution scan that made cached installs - # expensive. Package-level node_modules links are part of the live - # projection contract, so we fingerprint their resolved targets too. - for node_modules_dir in node_modules ${nodeModulesPaths}; do - if [ -d "$node_modules_dir" ]; then - printf 'dir %s\n' "$node_modules_dir" - else - printf 'missing %s\n' "$node_modules_dir" - continue - fi - - find "$node_modules_dir" -mindepth 1 -maxdepth 2 -type l -print \ - | LC_ALL=C sort \ - | while IFS= read -r link_path; do - link_target="$(readlink "$link_path" || true)" - if [ -e "$link_path" ]; then - printf 'link %s -> %s\n' "$link_path" "$link_target" - else - printf 'broken-link %s -> %s\n' "$link_path" "$link_target" - fi - done - done - - if [ -f node_modules/.modules.yaml ]; then - printf 'modules-yaml ' - sha256sum node_modules/.modules.yaml | awk '{print $1}' - else - printf 'modules-yaml missing\n' - fi - } | compute_hash + # Keep the warm-path fingerprint semantics identical while avoiding the + # shell pipeline's per-link process overhead. The helper hashes the same + # ordered line stream that the previous bash implementation produced. + NODE_MODULES_HELPER_MODE="projection-hash" \ + PNPM_ROOT_MODULES_YAML="node_modules/.modules.yaml" \ + NODE_MODULES_DIRS="$(printf '%s\n' node_modules ${nodeModulesPaths})" \ + ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} } ''; @@ -397,7 +371,7 @@ let fi fi - if [ "$_purged_node_modules" != true ] && ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionHealthScript} ${healthCheckNodeModulesPaths}; then + if [ "$_purged_node_modules" != true ] && ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} ${healthCheckNodeModulesPaths}; then echo "[pnpm] node_modules projection is stale, purging install state" purge_node_modules node_modules ${nodeModulesPaths} if [ -n "''${_gvs_links_dir:-}" ]; then @@ -419,7 +393,7 @@ let run_pnpm_install fi - if ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionHealthScript} ${healthCheckNodeModulesPaths}; then + if ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} ${healthCheckNodeModulesPaths}; then echo "[pnpm] node_modules projection is still unhealthy after install" >&2 exit 1 fi diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index b3f569824..4dcd581b3 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -157,8 +157,10 @@ if [ "${1:-}" = "install" ]; then echo "ERR_PNPM_META_FETCH_FAIL GET https://registry.npmjs.org/demo: request to https://registry.npmjs.org/demo failed, reason: Socket timeout" >&2 exit 42 fi - mkdir -p node_modules + mkdir -p node_modules vendor/pkg-v1 touch node_modules/.install-ok + printf '{"name":"pkg","version":"1.0.0"}\n' > vendor/pkg-v1/package.json + ln -snf ../vendor/pkg-v1 node_modules/pkg # The warm-path status now fingerprints the root projection metadata that # pnpm always writes on a real install. Keep the smoke fixture aligned with # that contract so the test still exercises the task logic instead of @@ -357,7 +359,23 @@ echo "Test 8: status hits after install with the default GVS path" assert_exit_code 0 "$exit_code" "status should hit after default-PNPM_HOME install" ) -echo "Test 9: status still hits when PNPM_HOME changes but store-dir stays shared" +echo "Test 9: outer cache hit misses when a projected symlink disappears" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + export DEVENV_SETUP_OUTER_CACHE_HIT=1 + bash "$tmpdir/pnpm-install.exec.sh" + rm -f node_modules/pkg + set +e + bash "$tmpdir/pnpm-install.status.sh" + exit_code=$? + set -e + unset DEVENV_SETUP_OUTER_CACHE_HIT + assert_exit_code 1 "$exit_code" "outer-hit status should miss when a projected symlink disappears" +) + +echo "Test 10: status still hits when PNPM_HOME changes but store-dir stays shared" ( cd "$workspace" export HOME="$tmpdir/home" @@ -369,7 +387,7 @@ echo "Test 9: status still hits when PNPM_HOME changes but store-dir stays share assert_exit_code 0 "$exit_code" "status should hit when only PNPM_HOME changes" ) -echo "Test 10: status misses after effective store-dir changes" +echo "Test 11: status misses after effective store-dir changes" ( cd "$workspace" export HOME="$tmpdir/home" @@ -382,10 +400,10 @@ echo "Test 10: status misses after effective store-dir changes" assert_exit_code 1 "$exit_code" "status should miss when store-dir changes" ) -echo "Test 11: exec invoked pnpm install" +echo "Test 12: exec invoked pnpm install" grep -q "^install " "$tmpdir/pnpm.log" -echo "Test 12: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" +echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" ( cd "$workspace" export HOME="$tmpdir/home" @@ -402,7 +420,7 @@ echo "Test 12: nested workspace exec uses its own cwd, cache, PNPM_HOME, and sha grep -qxF "npm_config_store_dir=$workspace/.direnv/pnpm-store" "$tmpdir/pnpm.log" ) -echo "Test 13: nested workspace status hits after nested install" +echo "Test 14: nested workspace status hits after nested install" ( cd "$workspace" export HOME="$tmpdir/home" @@ -416,7 +434,7 @@ echo "Test 13: nested workspace status hits after nested install" assert_exit_code 0 "$exit_code" "nested status should hit after nested install" ) -echo "Test 14: install flags and pre-install hooks are applied" +echo "Test 15: install flags and pre-install hooks are applied" ( cd "$workspace" export HOME="$tmpdir/home" @@ -430,7 +448,7 @@ echo "Test 14: install flags and pre-install hooks are applied" grep -qxF "install --config.confirmModulesPurge=false --config.store-dir=$workspace/.direnv/pnpm-store --ignore-scripts --config.public-hoist-pattern=*" "$tmpdir/pnpm.log" ) -echo "Test 15: CI install failures preserve and classify the pnpm log" +echo "Test 16: CI install failures preserve and classify the pnpm log" ( cd "$workspace" export HOME="$tmpdir/home" @@ -454,21 +472,21 @@ echo "Test 15: CI install failures preserve and classify the pnpm log" grep -qF "Socket timeout" <<< "$output" ) -echo "Test 16: generated test task runs vitest without pnpm exec" +echo "Test 17: generated test task runs vitest without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/test-demo.exec.sh")" [ "$output" = "vitest-shim:run" ] ) -echo "Test 17: generated storybook task runs storybook without pnpm exec" +echo "Test 18: generated storybook task runs storybook without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/storybook-demo.exec.sh")" [ "$output" = "storybook-shim:build" ] ) -echo "Test 18: clean leaves shared GVS links intact" +echo "Test 19: clean leaves shared GVS links intact" ( cd "$workspace" mkdir -p "$workspace/.direnv/pnpm-store/v11/links/shared-pkg" From f940b6bc20e539aeb746093a0b58737d9fc5f308 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Sat, 28 Mar 2026 15:39:59 +0100 Subject: [PATCH 11/14] Speed up setup fingerprint tool identity checks --- CHANGELOG.md | 1 + nix/devenv-modules/tasks/shared/setup.nix | 25 ++++++-- .../tasks/shared/tests/setup-cache.test.sh | 61 +++++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ba5eac0..2885412d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,7 @@ All notable changes to this project will be documented in this file. - **devenv/tasks**: make warm shell bootstrap commit-scoped and remove `ts:emit` from shell entry - Adds an outer `setup:auto` cache so warm `devenv shell` skips unchanged bootstrap work instead of traversing `pnpm:install`, `genie:run`, and `mr:apply` on every entry - Switches shell bootstrap from `mr:sync` to initial `mr:apply` so a fresh worktree is normalized without fetching on every shell + - Replaces setup fingerprint tool-version probes with resolved tool-identity hashing so warm shells do not pay `pnpm`, `genie`, or `mr` CLI startup just to validate unchanged setup inputs - Speeds up warm task status paths by using direct `mr status`, fingerprint-based `genie:run` caching, a one-process `pnpm:install` projection hash that preserves the previous structural guarantees, and a `ts:emit` graph that excludes `noEmit` references at emit time - Hardens the fast paths by making the outer cache only track setup inputs while each task still verifies its own outputs before skipping - **devenv/otel**: pin `devenv` temporarily to the post-#2661 upstream commit and move OTEL shell-entry notices onto `devenv.messages` diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index 21db76dbc..e0954d1b4 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -151,6 +151,26 @@ let allSetupTasks = setupTasks; setupFingerprintEnv = '' compute_setup_fingerprint() { + resolve_setup_tool_identity() { + _setup_tool="$1" + command -v "$_setup_tool" >/dev/null 2>&1 || return 0 + + _setup_tool_path="$(command -v "$_setup_tool")" + _setup_tool_resolved="$(${pkgs.coreutils}/bin/realpath "$_setup_tool_path" 2>/dev/null || printf '%s\n' "$_setup_tool_path")" + + printf 'tool %s path %s\n' "$_setup_tool" "$_setup_tool_path" + printf 'tool %s resolved %s\n' "$_setup_tool" "$_setup_tool_resolved" + + # Resolved Nix store paths already identify an immutable tool build. For + # mutable shims outside the store, hash the resolved target so upgrades + # still invalidate setup without paying each CLI's startup cost. + if [ -f "$_setup_tool_resolved" ] && [[ "$_setup_tool_resolved" != /nix/store/* ]]; then + printf 'tool %s sha256 %s\n' \ + "$_setup_tool" \ + "$(${pkgs.coreutils}/bin/sha256sum "$_setup_tool_resolved" | awk '{print $1}')" + fi + } + # This outer fingerprint exists because devenv's built-in `status` # semantics do not prune a dependency subtree: the scheduler only runs a # task's status command once that task itself is ready to execute, after @@ -211,10 +231,7 @@ let # changing the active pnpm/genie/mr binary still invalidates the outer # cache and forces the next shell to re-validate or refresh setup. for _setup_tool in pnpm genie mr; do - if command -v "$_setup_tool" >/dev/null 2>&1; then - printf 'tool %s path %s\n' "$_setup_tool" "$(command -v "$_setup_tool")" - printf 'tool %s version %s\n' "$_setup_tool" "$($_setup_tool --version 2>/dev/null | ${pkgs.coreutils}/bin/head -n1 || echo unknown)" - fi + resolve_setup_tool_identity "$_setup_tool" done for _setup_file in package.json pnpm-workspace.yaml pnpm-lock.yaml .npmrc megarepo.kdl megarepo.json megarepo.lock; do diff --git a/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh b/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh index d9d65171a..bb7dce413 100755 --- a/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/setup-cache.test.sh @@ -32,6 +32,29 @@ simulate_setup_outer_cache_hit() { [ "$current_fingerprint" = "$cached" ] } +simulate_tool_identity() { + local tool_name="$1" + local tool_path="$2" + local resolved_path + + resolved_path=$(python - <<'PY' "$tool_path" +import pathlib +import sys + +print(pathlib.Path(sys.argv[1]).resolve()) +PY +) + + { + printf 'tool %s path %s\n' "$tool_name" "$tool_path" + printf 'tool %s resolved %s\n' "$tool_name" "$resolved_path" + + if [ -f "$resolved_path" ] && [[ "$resolved_path" != /nix/store/* ]]; then + printf 'tool %s sha256 %s\n' "$tool_name" "$(shasum -a 256 "$resolved_path" | awk '{print $1}')" + fi + } | shasum -a 256 | awk '{print $1}' +} + echo "Running setup-cache tests..." echo "" @@ -101,5 +124,43 @@ exit_code=$? set -e assert_exit_code 1 "$exit_code" "different fingerprint text returns 1 (needs to run)" +echo "" +echo "Test 8: Mutable tool target content invalidates fingerprint" +tool_dir="$test_dir/tool" +mkdir -p "$tool_dir/bin" "$tool_dir/pkg-v1" "$tool_dir/pkg-v2" +printf 'echo v1\n' > "$tool_dir/pkg-v1/tool" +printf 'echo v2\n' > "$tool_dir/pkg-v2/tool" +chmod +x "$tool_dir/pkg-v1/tool" "$tool_dir/pkg-v2/tool" +ln -s ../pkg-v1/tool "$tool_dir/bin/tool" + +tool_fp_v1=$(simulate_tool_identity tool "$tool_dir/bin/tool") +ln -sf ../pkg-v2/tool "$tool_dir/bin/tool" +tool_fp_v2=$(simulate_tool_identity tool "$tool_dir/bin/tool") + +if [ "$tool_fp_v1" = "$tool_fp_v2" ]; then + echo "FAIL: retargeting mutable tool should change fingerprint" + exit 1 +fi +echo " ok: retargeting mutable tool changes fingerprint" + +echo "" +echo "Test 9: Nix store style tool path fingerprints by resolved path" +store_dir="$test_dir/nix/store/hash-demo-tool/bin" +mkdir -p "$store_dir" +printf 'echo store-tool\n' > "$store_dir/tool" +chmod +x "$store_dir/tool" +ln -s "$store_dir/tool" "$tool_dir/bin/store-tool" + +store_fp_1=$(simulate_tool_identity store-tool "$tool_dir/bin/store-tool") +mv "$test_dir/nix/store/hash-demo-tool" "$test_dir/nix/store/hash-demo-tool-2" +ln -sf "$test_dir/nix/store/hash-demo-tool-2/bin/tool" "$tool_dir/bin/store-tool" +store_fp_2=$(simulate_tool_identity store-tool "$tool_dir/bin/store-tool") + +if [ "$store_fp_1" = "$store_fp_2" ]; then + echo "FAIL: changing resolved store path should change fingerprint" + exit 1 +fi +echo " ok: resolved store path change invalidates fingerprint" + echo "" echo "All setup-cache tests passed" From 3134ed052c40e8a0bebfd5c2efd6a04ebfb9d885 Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Sat, 28 Mar 2026 16:30:38 +0100 Subject: [PATCH 12/14] Harden task cache review fixes --- nix/devenv-modules/tasks/shared/genie.nix | 7 +- nix/devenv-modules/tasks/shared/pnpm.nix | 4 + .../tasks/shared/tests/ts-task-smoke.test.sh | 232 ++++++++++++++++++ nix/devenv-modules/tasks/shared/ts.nix | 29 ++- 4 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 nix/devenv-modules/tasks/shared/tests/ts-task-smoke.test.sh diff --git a/nix/devenv-modules/tasks/shared/genie.nix b/nix/devenv-modules/tasks/shared/genie.nix index df348b7d2..381fb92b1 100644 --- a/nix/devenv-modules/tasks/shared/genie.nix +++ b/nix/devenv-modules/tasks/shared/genie.nix @@ -24,6 +24,8 @@ let generatedFilesFile = "${cacheRoot}/generated-files.txt"; collectGenieGeneratedFiles = '' collect_genie_generated_files() { + # Genie owns these markers, so the warm-path fingerprint follows the same + # explicit generated-file contract as the generator itself. ${pkgs.ripgrep}/bin/rg -l \ --glob '!tmp/**' \ --glob '!.git/**' \ @@ -99,7 +101,10 @@ let if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then # The outer setup fingerprint already covers tracked generated-file # drift plus genie binary identity. On that warm path, only prove that - # the outputs we generated last time still exist. + # the outputs we generated last time still exist. Content drift is + # intentionally deferred to the next full fingerprint recomputation so + # shell entry does not have to boot the generator or re-hash every + # generated file on every hit. [ -f ${lib.escapeShellArg stateFile} ] || exit 1 [ -f ${lib.escapeShellArg generatedFilesFile} ] || exit 1 while IFS= read -r file; do diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 54ef8e128..46499e5c8 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -423,6 +423,10 @@ let fi if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then + # Keep shell entry fast by reusing the cached install-state proof and + # only re-validating the realized projection structure here. The full + # semantic health check still runs in the exec path before install can + # be treated as clean again. ${computeProjectionStateHashFn} current_projection_hash="$(compute_projection_state_hash)" stored_projection_hash="$(cat "$projection_hash_file")" diff --git a/nix/devenv-modules/tasks/shared/tests/ts-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/ts-task-smoke.test.sh new file mode 100644 index 000000000..a7b17ec35 --- /dev/null +++ b/nix/devenv-modules/tasks/shared/tests/ts-task-smoke.test.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$TESTS_DIR/../../../../.." && pwd)" + +assert_exit_code() { + local expected="$1" + local actual="$2" + local label="$3" + + if [ "$expected" != "$actual" ]; then + echo "FAIL: $label" + echo " expected exit code: $expected" + echo " actual exit code: $actual" + exit 1 + fi +} + +extract_ts_emit_script() { + local attr="$1" + local output_path="$2" + + nix eval --impure --raw --expr " + let + flake = builtins.getFlake (toString $ROOT); + pkgs = import flake.inputs.nixpkgs { system = builtins.currentSystem; }; + evaluated = pkgs.lib.evalModules { + modules = [ + ({ ... }: { + options.tasks = pkgs.lib.mkOption { type = pkgs.lib.types.attrsOf pkgs.lib.types.anything; default = { }; }; + options.processes = pkgs.lib.mkOption { type = pkgs.lib.types.attrsOf pkgs.lib.types.anything; default = { }; }; + options.packages = pkgs.lib.mkOption { type = pkgs.lib.types.listOf pkgs.lib.types.anything; default = [ ]; }; + }) + ((import $ROOT/nix/devenv-modules/tasks/shared/ts.nix { + tsconfigFile = \"tsconfig.all.json\"; + tscBin = \"tsc\"; + }) { + pkgs = pkgs; + lib = pkgs.lib; + config = { }; + }) + ]; + }; + in evaluated.config.tasks.\"ts:emit\".${attr} + " > "$output_path" + chmod +x "$output_path" +} + +echo "Running ts task smoke test..." +echo "" + +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +workspace="$tmpdir/workspace" +mkdir -p \ + "$workspace/node_modules/typescript" \ + "$workspace/packages/no-emit" \ + "$workspace/packages/emit" \ + "$tmpdir/bin" + +cat > "$workspace/tsconfig.all.json" <<'EOF' +{ + // Root-level comment should be ignored + "files": [], + "references": [ + { "path": "packages/no-emit/tsconfig.json" }, // explicit file path + // This mid-file comment used to break the old JSON.parse path. + { "path": "packages/emit" } + ] +} +EOF + +cat > "$workspace/packages/no-emit/tsconfig.json" <<'EOF' +{ + "compilerOptions": { + // This comment is intentionally mid-file. + "composite": true, + "noEmit": true + } +} +EOF + +cat > "$workspace/packages/emit/tsconfig.json" <<'EOF' +{ + "compilerOptions": { + "composite": true, + // Keep this project in the emit graph. + "declaration": true + } +} +EOF + +cat > "$workspace/node_modules/typescript/package.json" <<'EOF' +{"name":"typescript","main":"./index.js"} +EOF + +cat > "$workspace/node_modules/typescript/index.js" <<'EOF' +const stripLineComments = (source) => { + let result = '' + let inString = false + let escaped = false + + for (let index = 0; index < source.length; index += 1) { + const char = source[index] + const next = source[index + 1] + + if (inString) { + result += char + if (escaped) { + escaped = false + } else if (char === '\\\\') { + escaped = true + } else if (char === '"') { + inString = false + } + continue + } + + if (char === '"') { + inString = true + result += char + continue + } + + if (char === '/' && next === '/') { + while (index < source.length && source[index] !== '\n') { + index += 1 + } + if (index < source.length) { + result += '\n' + } + continue + } + + result += char + } + + return result +} + +const parseJsonc = (source) => + JSON.parse( + stripLineComments(source) + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/,\s*([}\]])/g, '$1') + ) + +exports.readConfigFile = (filePath, readFile) => { + try { + return { config: parseJsonc(readFile(filePath)) } + } catch (error) { + return { error: { messageText: String(error.message ?? error) } } + } +} +EOF + +cat > "$tmpdir/bin/tsc" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "${TEST_TSC_LOG:?}" + +config_path="" +prev="" +for arg in "$@"; do + if [ "$prev" = "--build" ]; then + config_path="$arg" + break + fi + prev="$arg" +done + +if [ -z "$config_path" ]; then + echo "missing --build tsconfig path" >&2 + exit 1 +fi + +TEST_CAPTURED_TSCONFIG="${TEST_CAPTURED_TSCONFIG:?}" \ +node - "$config_path" <<'NODE' +const fs = require('node:fs') + +const [configPath] = process.argv.slice(2) +const config = JSON.parse(fs.readFileSync(configPath, 'utf8')) + +if (!Array.isArray(config.references)) { + throw new Error('references missing from generated emit tsconfig') +} + +const paths = config.references.map((reference) => reference.path) +if (paths.includes('packages/no-emit/tsconfig.json')) { + throw new Error('noEmit project should be removed from generated emit tsconfig') +} +if (!paths.includes('packages/emit')) { + throw new Error('emit project should remain in generated emit tsconfig') +} + +fs.copyFileSync(configPath, process.env.TEST_CAPTURED_TSCONFIG) +NODE +EOF +chmod +x "$tmpdir/bin/tsc" + +extract_ts_emit_script "exec" "$tmpdir/ts-emit.exec.sh" +extract_ts_emit_script "status" "$tmpdir/ts-emit.status.sh" + +export PATH="$tmpdir/bin:$PATH" +export TEST_TSC_LOG="$tmpdir/tsc.log" +export TEST_CAPTURED_TSCONFIG="$tmpdir/captured-tsconfig.json" + +echo "Test 1: ts:emit exec filters noEmit refs even with inline comments" +( + cd "$workspace" + bash "$tmpdir/ts-emit.exec.sh" +) +test -f "$TEST_CAPTURED_TSCONFIG" + +echo "Test 2: ts:emit status uses the same filtered graph" +( + cd "$workspace" + : > "$TEST_TSC_LOG" + rm -f "$TEST_CAPTURED_TSCONFIG" + set +e + bash "$tmpdir/ts-emit.status.sh" + exit_code=$? + set -e + assert_exit_code 0 "$exit_code" "ts:emit status should succeed for an already-clean filtered graph" +) +test -f "$TEST_CAPTURED_TSCONFIG" +grep -q -- '--dry --noCheck --verbose --pretty false' "$TEST_TSC_LOG" + +echo "" +echo "ts task smoke test passed" diff --git a/nix/devenv-modules/tasks/shared/ts.nix b/nix/devenv-modules/tasks/shared/ts.nix index 016987ce3..49a2f1e88 100644 --- a/nix/devenv-modules/tasks/shared/ts.nix +++ b/nix/devenv-modules/tasks/shared/ts.nix @@ -67,9 +67,28 @@ const path = require('node:path') const [sourceTsconfig, targetTsconfig] = process.argv.slice(2) -const readJsonWithLeadingComments = (filePath) => { - const contents = fs.readFileSync(filePath, 'utf8') - return JSON.parse(contents.replace(/^(?:\s*\/\/.*\n)+/, "")) +const loadTypescript = () => { + try { + return require(require.resolve('typescript', { paths: [path.dirname(sourceTsconfig), process.cwd()] })) + } catch (error) { + throw new Error( + 'Unable to resolve TypeScript while preparing ts:emit: ' + + String(error?.message ?? error) + ) + } +} + +const typescript = loadTypescript() + +const readTsconfig = (filePath) => { + const parsed = typescript.readConfigFile(filePath, (path) => fs.readFileSync(path, 'utf8')) + if (parsed.error) { + const message = typeof parsed.error.messageText === 'string' + ? parsed.error.messageText + : JSON.stringify(parsed.error.messageText) + throw new Error('Failed to parse ' + filePath + ': ' + message) + } + return parsed.config } const resolveReferenceTsconfig = (referencePath) => { @@ -77,7 +96,7 @@ const resolveReferenceTsconfig = (referencePath) => { return path.extname(resolvedPath) ? resolvedPath : path.join(resolvedPath, 'tsconfig.json') } -const rootConfig = readJsonWithLeadingComments(sourceTsconfig) +const rootConfig = readTsconfig(sourceTsconfig) const baseDir = path.dirname(sourceTsconfig) rootConfig.references = (rootConfig.references ?? []).filter((reference) => { @@ -86,7 +105,7 @@ rootConfig.references = (rootConfig.references ?? []).filter((reference) => { return true } - const refConfig = readJsonWithLeadingComments(refTsconfig) + const refConfig = readTsconfig(refTsconfig) return refConfig.compilerOptions?.noEmit !== true }) From bb0021d601ab8d974447ecbac2333a4ab42f23bd Mon Sep 17 00:00:00 2001 From: Johannes Schickling Date: Wed, 1 Apr 2026 11:12:26 +0200 Subject: [PATCH 13/14] fix(ci): repair pinned devenv resolution step --- .github/workflows/ci.yml | 29 +++++++++-------------------- genie/ci-workflow.ts | 4 +--- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd018e24a..099c12bdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,13 +73,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -240,8 +238,9 @@ jobs: return 0 } - __flattened=$(perl -0pe 's/\e\[[0-9;]*m//g; s/\n/ /g' "$__log") - __path=$(printf '%s' "$__flattened" | grep -oP "error:\s+path '\K/nix/store/[^']*(?='\s+is not valid)" 2>/dev/null | head -1 | tr -d '[:space:]' || true) + __flattened=$(tr ' + ' ' ' < "$__log" | sed "s/$(printf '\033')\[[0-9;]*m//g") + __path=$(printf '%s' "$__flattened" | sed -n "s#.*error:[[:space:]]*path '\\(/nix/store/[^']*\\)'[[:space:]]*is not valid.*#\\1#p" | head -1 | tr -d '[:space:]' || true) __saw_invalid_path=false __saw_cachix_signature=false [ -n "$__path" ] && __saw_invalid_path=true @@ -507,13 +506,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -841,13 +838,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -1175,13 +1170,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -1486,13 +1479,11 @@ jobs: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -1652,13 +1643,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 5f87d5af0..722115c64 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -1232,9 +1232,7 @@ nix run "github:overengineeringstudio/effect-utils/$EU_REV#megarepo" -- apply -- */ export const validateNixStoreStep = { name: 'Resolve devenv', - run: `if [ -z "${'${DEVENV_REV:-}'}" ]; then - ${resolveDevenvRevScript} -fi + run: `${resolveDevenvRevScript} ${resolveDevenvFnScript} From efb84ac7bcc15b8afecbbf4cd656005513e7a5fc Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Fri, 8 May 2026 16:05:40 +0200 Subject: [PATCH 14/14] Revive devenv perf work on v2.1 --- .github/workflows/ci.yml | 154 +++++----- .gitignore | 1 + CHANGELOG.md | 3 +- context/workarounds/devenv-issues.md | 6 +- devenv.lock | 275 +++++++++++++++--- devenv.nix | 5 - devenv.yaml | 5 +- genie/ci-workflow.ts | 4 - nix/devenv-modules/otel.nix | 27 +- .../check-node-modules-projection-health.cjs | 2 + nix/devenv-modules/tasks/shared/megarepo.nix | 2 +- nix/devenv-modules/tasks/shared/pnpm.nix | 1 + nix/devenv-modules/tasks/shared/setup.nix | 3 +- .../shared/tests/pnpm-task-smoke.test.sh | 16 +- nix/devenv-modules/tasks/shared/ts.nix | 96 +++--- 15 files changed, 399 insertions(+), 201 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 099c12bdd..8350205f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,96 +185,114 @@ jobs: - name: Verify OTEL shell entry shell: bash run: | - __nix_gc_retry() { - local __task='devenv tasks run otel:test --mode before' __max=${NIX_GC_RACE_MAX_RETRIES:-10} __heartbeat=${CI_PROGRESS_HEARTBEAT_SECONDS:-60} __n=1 __log __rc __path __start __now __elapsed __hb_pid __flattened __saw_invalid_path __saw_cachix_signature - __start=$(date +%s) + __nix_gc_retry_helper=$(mktemp) + cat > "$__nix_gc_retry_helper" <<'EOF' + #!/usr/bin/env bash + + run_nix_gc_race_retry() { + local task="$1" + local command="$2" + local max="${NIX_GC_RACE_MAX_RETRIES:-10}" + local heartbeat="${CI_PROGRESS_HEARTBEAT_SECONDS:-60}" + local attempt=1 + local log rc path start now elapsed hb_pid flattened saw_invalid_path saw_cachix_signature had_errexit - __write_summary() { + start="$(date +%s)" + + write_summary() { [ -n "${GITHUB_STEP_SUMMARY:-}" ] || return 0 { echo "### CI Task" - # Keep summary values plain text. Backticks inside double quotes trigger - # shell command substitution and turned failed-task metadata into bogus - # commands on GitHub Actions runners. - echo "- Task: $__task" + echo "- Task: $task" echo "- Status: $1" - echo "- Duration: $__elapsed s" - echo "- Attempts: $__n/$__max" + echo "- Duration: $elapsed s" + echo "- Attempts: $attempt/$max" [ -z "${2:-}" ] || echo "- Note: $2" } >> "$GITHUB_STEP_SUMMARY" } - while [ "$__n" -le "$__max" ]; do - echo "::notice::[ci] starting $__task (attempt $__n/$__max)" + while [ "$attempt" -le "$max" ]; do + echo "::notice::[ci] starting $task (attempt $attempt/$max)" ( - while sleep "$__heartbeat"; do - __now=$(date +%s) - __elapsed=$((__now - __start)) - echo "::notice::[ci] $__task still running after $__elapsed s (attempt $__n/$__max)" + while sleep "$heartbeat"; do + now=$(date +%s) + elapsed=$((now - start)) + echo "::notice::[ci] $task still running after $elapsed s (attempt $attempt/$max)" done ) & - __hb_pid=$! + hb_pid=$! - __log=$(mktemp) + log=$(mktemp) + had_errexit=false + case $- in + *e*) had_errexit=true ;; + esac set +e - eval "$1" > >(tee -a "$__log") 2> >(tee -a "$__log" >&2) - __rc=$? - set -e + eval "$command" > >(tee -a "$log") 2> >(tee -a "$log" >&2) + rc=$? + if [ "$had_errexit" = true ]; then + set -e + fi - kill "$__hb_pid" 2>/dev/null || true - wait "$__hb_pid" 2>/dev/null || true + kill "$hb_pid" 2>/dev/null || true + wait "$hb_pid" 2>/dev/null || true - __now=$(date +%s) - __elapsed=$((__now - __start)) + now=$(date +%s) + elapsed=$((now - start)) - [ $__rc -eq 0 ] && { - echo "::notice::[ci] completed $__task in $__elapsed s" - if [ "$__n" -gt 1 ]; then - __write_summary success "Recovered from Nix GC race after retry" + if [ "$rc" -eq 0 ]; then + echo "::notice::[ci] completed $task in $elapsed s" + if [ "$attempt" -gt 1 ]; then + write_summary success "Recovered from Nix GC race after retry" else - __write_summary success + write_summary success fi - rm -f "$__log" + rm -f "$log" return 0 - } + fi + + flattened=$(tr '\r\n' ' ' < "$log" | sed -E $'s/\x1B\[[0-9;]*m//g') + path=$(printf '%s' "$flattened" | + grep -o "error:[[:space:]]*path '/nix/store/[^']*'[[:space:]]*is not valid" | + head -1 | + grep -o "/nix/store/[^']*" | + tr -d '[:space:]' || true) + saw_invalid_path=false + saw_cachix_signature=false + [ -n "$path" ] && saw_invalid_path=true + printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*Failed to convert config\.cachix to JSON' && saw_cachix_signature=true || true + printf '%s' "$flattened" | grep -Eq 'error:[[:space:]]*.*while evaluating the option.*cachix\.package' && saw_cachix_signature=true || true + rm -f "$log" - __flattened=$(tr ' - ' ' ' < "$__log" | sed "s/$(printf '\033')\[[0-9;]*m//g") - __path=$(printf '%s' "$__flattened" | sed -n "s#.*error:[[:space:]]*path '\\(/nix/store/[^']*\\)'[[:space:]]*is not valid.*#\\1#p" | head -1 | tr -d '[:space:]' || true) - __saw_invalid_path=false - __saw_cachix_signature=false - [ -n "$__path" ] && __saw_invalid_path=true - printf '%s' "$__flattened" | grep -q 'Failed to convert config\.cachix to JSON' && __saw_cachix_signature=true || true - # Match the semantic signal, not the exact quote punctuation, so the shell - # stays valid even when the human-facing error wraps the option name. - printf '%s' "$__flattened" | grep -q 'while evaluating the option' && printf '%s' "$__flattened" | grep -q 'cachix\.package' && __saw_cachix_signature=true || true - rm -f "$__log" - if [ "$__saw_invalid_path" != true ] && [ "$__saw_cachix_signature" != true ]; then - echo "::warning::[ci] $__task failed after $__elapsed s without a detected Nix store validity race" - __write_summary failure "No Nix GC race signature detected" - return $__rc + if [ "$saw_invalid_path" != true ] && [ "$saw_cachix_signature" != true ]; then + echo "::warning::[ci] $task failed after $elapsed s without a detected Nix store validity race" + write_summary failure "No Nix GC race signature detected" + return "$rc" fi - if [ "$__saw_cachix_signature" = true ] && [ -n "$__path" ]; then - echo "::warning::Nix store validity race detected for $__task via cachix eval wrapper (attempt $__n/$__max): $__path" - elif [ "$__saw_cachix_signature" = true ]; then - # The cachix wrapper can surface the GC race before the invalid path makes - # it into the flattened log. Retrying after clearing the eval cache still - # recovers that case in practice. - echo "::warning::Nix store validity race detected for $__task via cachix eval wrapper without extracted store path (attempt $__n/$__max)" + + if [ "$saw_cachix_signature" = true ] && [ -n "$path" ]; then + echo "::warning::Nix store validity race detected for $task via cachix eval wrapper (attempt $attempt/$max): $path" + elif [ "$saw_cachix_signature" = true ]; then + echo "::warning::Nix store validity race detected for $task via cachix eval wrapper without extracted store path (attempt $attempt/$max)" else - echo "::warning::Nix store validity race detected for $__task (attempt $__n/$__max): $__path" + echo "::warning::Nix store validity race detected for $task (attempt $attempt/$max): $path" fi - [ -z "$__path" ] || nix-store --realise "$__path" 2>/dev/null || true + + [ -z "$path" ] || nix-store --realise "$path" 2>/dev/null || true rm -rf ~/.cache/nix/eval-cache-* - __n=$((__n + 1)) + attempt=$((attempt + 1)) done - __now=$(date +%s) - __elapsed=$((__now - __start)) - echo "::error::Nix GC race retry exhausted for $__task ($__max attempts)" - __write_summary failure "Nix GC race retry exhausted" + now=$(date +%s) + elapsed=$((now - start)) + echo "::error::Nix GC race retry exhausted for $task ($max attempts)" + write_summary failure "Nix GC race retry exhausted" return 1 - }; __nix_gc_retry 'if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" PNPM_HOME="${PNPM_HOME:-${{ github.workspace }}/.pnpm-home}" PNPM_STORE_DIR="${PNPM_STORE_DIR:-${{ runner.temp }}/pnpm-store/${{ github.job }}}" DT_PASSTHROUGH=1 "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run otel:test --mode before' + } + EOF + . "$__nix_gc_retry_helper" + rm -f "$__nix_gc_retry_helper" + run_nix_gc_race_retry 'devenv tasks run otel:test --mode before' 'if [ -n "${NIX_CONFIG:-}" ]; then NIX_CONFIG_WITH_APPEND=$(printf '"'"'%s\n%s'"'"' "$NIX_CONFIG" '"'"'restrict-eval = false'"'"'); else NIX_CONFIG_WITH_APPEND='"'"'restrict-eval = false'"'"'; fi; NIX_CONFIG="$NIX_CONFIG_WITH_APPEND" PNPM_HOME="${PNPM_HOME:-${{ github.workspace }}/.pnpm-home}" PNPM_STORE_DIR="${PNPM_STORE_DIR:-${{ runner.temp }}/pnpm-store/${{ github.job }}}" DT_PASSTHROUGH=1 "${DEVENV_BIN:?DEVENV_BIN not set}" tasks run otel:test --mode before' command -v script >/dev/null 2>&1 tmp_log="$(mktemp)" printf 'printf "OTEL_MODE=%%s\n" "$OTEL_MODE" @@ -1889,13 +1907,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -2115,13 +2131,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" @@ -2450,13 +2464,11 @@ jobs: key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - name: Resolve devenv run: | - if [ -z "${DEVENV_REV:-}" ]; then - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then echo '::error::devenv.lock missing .nodes.devenv.locked.rev' exit 1 fi - fi resolve_devenv() { nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" diff --git a/.gitignore b/.gitignore index bb320482a..78c0e7d2f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ tmp # Node.js node_modules +.pnpm-store # TypeScript *.tsbuildinfo diff --git a/CHANGELOG.md b/CHANGELOG.md index 2885412d8..c1e77c67c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,10 +100,9 @@ All notable changes to this project will be documented in this file. - Replaces setup fingerprint tool-version probes with resolved tool-identity hashing so warm shells do not pay `pnpm`, `genie`, or `mr` CLI startup just to validate unchanged setup inputs - Speeds up warm task status paths by using direct `mr status`, fingerprint-based `genie:run` caching, a one-process `pnpm:install` projection hash that preserves the previous structural guarantees, and a `ts:emit` graph that excludes `noEmit` references at emit time - Hardens the fast paths by making the outer cache only track setup inputs while each task still verifies its own outputs before skipping -- **devenv/otel**: pin `devenv` temporarily to the post-#2661 upstream commit and move OTEL shell-entry notices onto `devenv.messages` +- **devenv/otel**: update `devenv` to the upstream `v2.1` tag and move OTEL shell-entry notices onto `devenv.messages` - Resolves OTEL mode, dashboard sync, and Grafana trace-link construction in a dedicated shell-entry task instead of ad-hoc `enterShell` output - Auto-displays the OTEL shell-entry message through upstream task messages while keeping `otel-trace` as a lightweight re-open helper - - Adds a TODO to switch the temporary commit pin back to the `v2.0.7` tag once it is released - Scrubs ambient task trace context before emitting `devenv/shell:entry` so the shell root span cannot self-parent or collide with later `dt` root spans - Emits `devenv/shell:entry` via the pinned store path for `otel-span` so tracing still works before `enterShell` PATH mutations are fully visible - **@overeng/genie**: Validate GitHub Actions `runs-on` labels before emitting workflow YAML diff --git a/context/workarounds/devenv-issues.md b/context/workarounds/devenv-issues.md index ef1267b85..1812e19ca 100644 --- a/context/workarounds/devenv-issues.md +++ b/context/workarounds/devenv-issues.md @@ -144,7 +144,7 @@ devenv's PTY task runner sends two echo sentinels and reads until both are found - Emit OTEL shell-entry notices through `devenv.messages` task output. - Reuse the exported Grafana link env in `otel-trace` for on-demand reopening. -- Keep a TODO to return to the `v2.0.7` tag once that release is available. +- Use the upstream `v2.1` tag, which includes the task message support this flow needs. --- @@ -247,10 +247,6 @@ Git hooks run in a subprocess that doesn't inherit the direnv environment. - Remove manual JSON trace post-processing from CI pipelines - Update R10 status in this document to reflect full compliance -- **DEVENV-05 follow-up (tagged release contains #2661):** - - Replace the temporary commit pin with the `v2.0.7` tag - - Remove the temporary pin note from `devenv.yaml` / CI docs - - **COMPAT-01 improved (web coding agent support):** - When Claude Code Web adds Nix domains to allowlist: update status, remove "Full internet" workaround - When Codex fixes PATH persistence: update status, simplify setup scripts diff --git a/devenv.lock b/devenv.lock index c472b823b..7c5ff075b 100644 --- a/devenv.lock +++ b/devenv.lock @@ -19,11 +19,11 @@ ] }, "locked": { - "lastModified": 1767714506, - "narHash": "sha256-WaTs0t1CxhgxbIuvQ97OFhDTVUGd1HA+KzLZUZBhe0s=", + "lastModified": 1777487137, + "narHash": "sha256-TuvKVBX60mqyMT6OB5JqVEh1YIWtFMR/igLCaCdC9tw=", "owner": "cachix", "repo": "cachix", - "rev": "894c649f0daaa38bbcfb21de64be47dfa7cd0ec9", + "rev": "a66a440c321d35f7193472c317f42a55ccd1cb93", "type": "github" }, "original": { @@ -106,16 +106,17 @@ "pre-commit-hooks": "pre-commit-hooks_2" }, "locked": { - "lastModified": 1773440526, - "narHash": "sha256-OcX1MYqUdoalY3/vU67PEx8m6RvqGxX0LwKonjzXn7I=", - "owner": "nix-community", + "lastModified": 1772186516, + "narHash": "sha256-8s28pzmQ6TOIUzznwFibtW1CMieMUl1rYJIxoQYor58=", + "owner": "rossng", "repo": "crate2nix", - "rev": "e697d3049c909580128caa856ab8eb709556a97b", + "rev": "ba5dd398e31ee422fbe021767eb83b0650303a6e", "type": "github" }, "original": { - "owner": "nix-community", + "owner": "rossng", "repo": "crate2nix", + "rev": "ba5dd398e31ee422fbe021767eb83b0650303a6e", "type": "github" } }, @@ -155,24 +156,25 @@ "crate2nix": "crate2nix", "flake-compat": "flake-compat_3", "flake-parts": "flake-parts_3", + "ghostty": "ghostty", "git-hooks": "git-hooks_3", "nix": "nix", "nixd": "nixd", - "nixpkgs": "nixpkgs_4", + "nixpkgs": "nixpkgs_6", "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1774649847, - "narHash": "sha256-2h7rrOzLjyQdt20yHKPnK0fA+v0fj+whGaDBnmfGahY=", + "lastModified": 1777988467, + "narHash": "sha256-U7rb9FufadyCBLLsxVY6AJfy6TN24+uwaBBh8JVOP8s=", "owner": "cachix", "repo": "devenv", - "rev": "61170924d98492ad8842dca02ad8b912305d308b", + "rev": "2cf62a010000b70f15c78a72761fad7c9e6fb47a", "type": "github" }, "original": { "owner": "cachix", + "ref": "v2.1", "repo": "devenv", - "rev": "61170924d98492ad8842dca02ad8b912305d308b", "type": "github" } }, @@ -266,6 +268,22 @@ } }, "flake-compat_4": { + "flake": false, + "locked": { + "lastModified": 1761588595, + "narHash": "sha256-XKUZz9zewJNUj46b4AJdiRZJAvSZ0Dqj2BNfXvFlJC4=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "f387cd2afec9419c8ee37694406ca490c3f34ee5", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_5": { "flake": false, "locked": { "lastModified": 1767039857, @@ -334,11 +352,11 @@ ] }, "locked": { - "lastModified": 1772408722, - "narHash": "sha256-rHuJtdcOjK7rAHpHphUb1iCvgkU3GpfvicLMwwnfMT0=", + "lastModified": 1777678872, + "narHash": "sha256-EPIFsulyon7Z1vLQq5Fk64GR8L7cQsT+IPhcsukVbgk=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "f20dc5d9b8027381c474144ecabc9034d6a839a3", + "rev": "5250617bffd85403b14dbf43c3870e7f255d2c16", "type": "github" }, "original": { @@ -349,7 +367,7 @@ }, "flake-utils": { "inputs": { - "systems": "systems" + "systems": "systems_2" }, "locked": { "lastModified": 1731533236, @@ -365,6 +383,29 @@ "type": "github" } }, + "ghostty": { + "inputs": { + "flake-compat": "flake-compat_4", + "home-manager": "home-manager", + "nixpkgs": "nixpkgs_4", + "systems": "systems", + "zig": "zig", + "zon2nix": "zon2nix" + }, + "locked": { + "lastModified": 1777773742, + "narHash": "sha256-dZFc+8az7BUIs8+v45XqNnY5G6oXEwVfVVHZQuATSGQ=", + "owner": "ghostty-org", + "repo": "ghostty", + "rev": "1547dd667ab6d1f4ebcdc7282adc54c95752ee67", + "type": "github" + }, + "original": { + "owner": "ghostty-org", + "repo": "ghostty", + "type": "github" + } + }, "git-hooks": { "inputs": { "flake-compat": [ @@ -440,11 +481,11 @@ ] }, "locked": { - "lastModified": 1772893680, - "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=", + "lastModified": 1776796298, + "narHash": "sha256-PcRvlWayisPSjd0UcRQbhG8Oqw78AcPE6x872cPRHN8=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9", + "rev": "3cfd774b0a530725a077e17354fbdb87ea1c4aad", "type": "github" }, "original": { @@ -455,7 +496,7 @@ }, "git-hooks_4": { "inputs": { - "flake-compat": "flake-compat_4", + "flake-compat": "flake-compat_5", "gitignore": "gitignore_6", "nixpkgs": [ "nixpkgs" @@ -614,6 +655,28 @@ "type": "github" } }, + "home-manager": { + "inputs": { + "nixpkgs": [ + "devenv", + "ghostty", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1770586272, + "narHash": "sha256-Ucci8mu8QfxwzyfER2DQDbvW9t1BnTUJhBmY7ybralo=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "b1f916ba052341edc1f80d4b2399f1092a4873ca", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, "nix": { "inputs": { "flake-compat": [ @@ -640,16 +703,16 @@ ] }, "locked": { - "lastModified": 1774103430, - "narHash": "sha256-MRNVInSmvhKIg3y0UdogQJXe+omvKijGszFtYpd5r9k=", + "lastModified": 1776511668, + "narHash": "sha256-g2KEBuHpc3a56c+jPcg0+w6LSuIj6f+zzdztLCOyIhc=", "owner": "cachix", "repo": "nix", - "rev": "e127c1c94cefe02d8ca4cca79ef66be4c527510e", + "rev": "42d4b7de21c15f28c568410f4383fa06a8458a40", "type": "github" }, "original": { "owner": "cachix", - "ref": "devenv-2.32", + "ref": "devenv-2.34", "repo": "nix", "type": "github" } @@ -692,18 +755,15 @@ "devenv", "flake-parts" ], - "nixpkgs": [ - "devenv", - "nixpkgs" - ], + "nixpkgs": "nixpkgs_5", "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1773634079, - "narHash": "sha256-49qb4QNMv77VOeEux+sMd0uBhPvvHgVc0r938Bulvbo=", + "lastModified": 1777345723, + "narHash": "sha256-BhY3D5DhpDnnUcaY+AL/cpyYX+OIjQgnAkbPLZ08C38=", "owner": "nix-community", "repo": "nixd", - "rev": "8ecf93d4d93745e05ea53534e8b94f5e9506e6bd", + "rev": "6bf30951a3dc407a798d30db427e3f96ac9b39f5", "type": "github" }, "original": { @@ -731,11 +791,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1773597492, - "narHash": "sha256-hQ284SkIeNaeyud+LS0WVLX+WL2rxcVZLFEaK0e03zg=", + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a07d4ce6bee67d7c838a8a5796e75dff9caa21ef", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", "type": "github" }, "original": { @@ -794,15 +854,41 @@ } }, "nixpkgs_4": { + "locked": { + "lastModified": 1770537093, + "narHash": "sha256-XV30uo8tXuxdzuV8l3sojmlPRLd/8tpMsOp4lNzLGUo=", + "rev": "fef9403a3e4d31b0a23f0bacebbec52c248fbb51", + "type": "tarball", + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.05pre942631.fef9403a3e4d/nixexprs.tar.xz" + }, + "original": { + "type": "tarball", + "url": "https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.xz" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1776877367, + "narHash": "sha256-wMN1gM00sUQ2KC9CNr/XEOGdfOrl67PabIRv9AYayTo=", + "rev": "0726a0ecb6d4e08f6adced58726b95db924cef57", + "type": "tarball", + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre985613.0726a0ecb6d4/nixexprs.tar.xz" + }, + "original": { + "type": "tarball", + "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz" + } + }, + "nixpkgs_6": { "inputs": { "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1773704619, - "narHash": "sha256-LKtmit8Sr81z8+N2vpIaN/fyiQJ8f7XJ6tMSKyDVQ9s=", + "lastModified": 1776852779, + "narHash": "sha256-WwO/ITisCXwyiRgtktZgv3iGhAGO+IB5Av4kKCwezR0=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "906534d75b0e2fe74a719559dfb1ad3563485f43", + "rev": "ec3063523dcd911aeadb50faa589f237cdab5853", "type": "github" }, "original": { @@ -812,7 +898,7 @@ "type": "github" } }, - "nixpkgs_5": { + "nixpkgs_7": { "locked": { "lastModified": 1774106199, "narHash": "sha256-US5Tda2sKmjrg2lNHQL3jRQ6p96cgfWh3J1QBliQ8Ws=", @@ -929,7 +1015,7 @@ "inputs": { "devenv": "devenv", "git-hooks": "git-hooks_4", - "nixpkgs": "nixpkgs_5", + "nixpkgs": "nixpkgs_7", "playwright": "playwright", "tsgo": "tsgo" } @@ -942,11 +1028,11 @@ ] }, "locked": { - "lastModified": 1773630837, - "narHash": "sha256-zJhgAGnbVKeBMJOb9ctZm4BGS/Rnrz+5lfSXTVah4HQ=", + "lastModified": 1777778183, + "narHash": "sha256-Lqv9MZO0XAGcMbXJU+ULBSMD41Pf391uJehylUQKe7Y=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "f600ea449c7b5bb596fa1cf21c871cc5b9e31316", + "rev": "dbba5f888c82ef3ce594c451c33ac2474eb80847", "type": "github" }, "original": { @@ -956,6 +1042,22 @@ } }, "systems": { + "flake": false, + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { "locked": { "lastModified": 1681028828, "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", @@ -979,11 +1081,11 @@ ] }, "locked": { - "lastModified": 1772660329, - "narHash": "sha256-IjU1FxYqm+VDe5qIOxoW+pISBlGvVApRjiw/Y/ttJzY=", + "lastModified": 1775636079, + "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "3710e0e1218041bbad640352a0440114b1e10428", + "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba", "type": "github" }, "original": { @@ -1048,8 +1150,87 @@ "rev": "2a3bed2b4265fa1173c88771a21ce044e6480f75", "type": "github" } + }, + "zig": { + "inputs": { + "flake-compat": [ + "devenv", + "ghostty", + "flake-compat" + ], + "nixpkgs": [ + "devenv", + "ghostty", + "nixpkgs" + ], + "systems": [ + "devenv", + "ghostty", + "systems" + ] + }, + "locked": { + "lastModified": 1776789209, + "narHash": "sha256-G6B7Q4TXn7MZ1mB+f9rymjsYF5PLWoSvmbxijb/99bw=", + "owner": "mitchellh", + "repo": "zig-overlay", + "rev": "14fe971844e841297ddd2ce9783d6892b467af39", + "type": "github" + }, + "original": { + "owner": "mitchellh", + "repo": "zig-overlay", + "type": "github" + } + }, + "zig_2": { + "inputs": { + "nixpkgs": [ + "devenv", + "ghostty", + "zon2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1777234348, + "narHash": "sha256-fKw44a4qbUuI5eTG8k0gPbqMV5TOrjYF35PBzsYgd2U=", + "ref": "refs/heads/main", + "rev": "2c781c0609ecda600ab98f98cca417bbd981bd53", + "revCount": 1677, + "type": "git", + "url": "https://codeberg.org/jcollie/zig-overlay.git" + }, + "original": { + "type": "git", + "url": "https://codeberg.org/jcollie/zig-overlay.git" + } + }, + "zon2nix": { + "inputs": { + "nixpkgs": [ + "devenv", + "ghostty", + "nixpkgs" + ], + "zig": "zig_2" + }, + "locked": { + "lastModified": 1777314365, + "narHash": "sha256-eLxQaD0wc96Neqkln8wHS0rNq/chPODifFkhwrwilEU=", + "owner": "jcollie", + "repo": "zon2nix", + "rev": "a5a1d412ad1ab6305511997bbc92b3a9dd6cb784", + "type": "github" + }, + "original": { + "owner": "jcollie", + "ref": "main", + "repo": "zon2nix", + "type": "github" + } } }, "root": "root", "version": 7 -} +} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix index 92cf42b91..7a89e311a 100644 --- a/devenv.nix +++ b/devenv.nix @@ -353,11 +353,6 @@ in "genie:run" "mr:apply" ]; - innerCacheDirs = [ - "pnpm-install" - "genie-run" - "mr-apply" - ]; completionsCliNames = [ "genie" "mr" diff --git a/devenv.yaml b/devenv.yaml index 3c3d7c67b..d4cc28496 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,9 +1,6 @@ inputs: devenv: - # Temporary pin to the post-#2661 commit so shell-entry task messages are - # available before the upstream v2.0.7 release lands. - # TODO: Switch back to github:cachix/devenv/v2.0.7 once it is released. - url: github:cachix/devenv/61170924d98492ad8842dca02ad8b912305d308b + url: github:cachix/devenv/v2.1 nixpkgs: url: github:NixOS/nixpkgs/nixos-unstable git-hooks: diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 722115c64..bbaf243de 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -622,10 +622,6 @@ export const cachixStep = (opts: { name: string; authToken?: string }) => ({ /** * Prepare lock-pinned devenv metadata from devenv.lock. - * - * The lock may temporarily point at an upstream commit instead of a release tag - * while we validate a fix ahead of the next devenv release. - * TODO: Drop that temporary pin once v2.0.7 is available. */ export const preparePinnedDevenvStep = { name: 'Use pinned devenv from lock', diff --git a/nix/devenv-modules/otel.nix b/nix/devenv-modules/otel.nix index 0fe7c0bca..ee5ef46fc 100644 --- a/nix/devenv-modules/otel.nix +++ b/nix/devenv-modules/otel.nix @@ -390,11 +390,21 @@ let echo "[otel] ERROR: extraDashboards is not supported in OTEL_MODE=system" >&2 return 1 fi - if ! otel dash sync \ - --source "${allDashboards}" \ - --target "$OTEL_STATE_DIR/dashboards" >/dev/null 2>&1; then - echo "[otel] ERROR: otel dash sync failed" >&2 - return 1 + _otel_project_name="$(basename "''${DEVENV_ROOT:-devenv}")" + if otel dash sync --help >/dev/null 2>&1; then + if ! otel dash sync \ + --source "${allDashboards}" \ + --target "$OTEL_STATE_DIR/dashboards" >/dev/null 2>&1; then + echo "[otel] WARN: otel dash sync failed; continuing without refreshing dashboards" >&2 + fi + elif otel dash restore --help >/dev/null 2>&1; then + if ! otel dash restore \ + --project "$_otel_project_name" \ + --from "${allDashboards}" >/dev/null 2>&1; then + echo "[otel] WARN: otel dash restore failed; continuing without refreshing dashboards" >&2 + fi + else + echo "[otel] WARN: otel CLI does not support dashboard restore/sync; continuing without refreshing dashboards" >&2 fi _otel_mode_msg="[otel] Using system-level OTEL stack (mode=$OTEL_MODE)" else @@ -681,10 +691,11 @@ in lib.optionals (builtins.hasAttr "devenv:files:cleanup" config.tasks) [ "devenv:files:cleanup" ] ++ lib.optionals (builtins.hasAttr "devenv:files" config.tasks) [ "devenv:files" ] ++ [ "otel:shell-env" ] - ++ lib.optionals (builtins.hasAttr "setup:record-cache" config.tasks) [ "setup:record-cache@completed" ] + ++ lib.optionals (builtins.hasAttr "setup:record-cache" config.tasks) [ + "setup:record-cache@completed" + ] ++ lib.optionals ( - !(builtins.hasAttr "setup:record-cache" config.tasks) - && builtins.hasAttr "setup:gate" config.tasks + !(builtins.hasAttr "setup:record-cache" config.tasks) && builtins.hasAttr "setup:gate" config.tasks ) [ "setup:gate" ]; }; diff --git a/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs b/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs index cd638d23a..a7a908fb8 100644 --- a/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs +++ b/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs @@ -128,6 +128,8 @@ const runProjectionHash = () => { hash.update('\n') } + appendLine(`gvs-links-dir ${process.env.PNPM_GVS_LINKS_DIR || ''}`) + for (const nodeModulesDir of moduleDirs) { if (fs.existsSync(nodeModulesDir) && fs.statSync(nodeModulesDir).isDirectory()) { appendLine(`dir ${nodeModulesDir}`) diff --git a/nix/devenv-modules/tasks/shared/megarepo.nix b/nix/devenv-modules/tasks/shared/megarepo.nix index 6ee53d232..05194c181 100644 --- a/nix/devenv-modules/tasks/shared/megarepo.nix +++ b/nix/devenv-modules/tasks/shared/megarepo.nix @@ -42,7 +42,7 @@ let # Rewrite the manifest atomically so a failed `mr ls` never leaves behind # an empty file that would make the warm-path output proof vacuous. mr ls --output json \ - | ${jq} -r 'select(._tag == "Success") | .value.members[].name' \ + | ${jq} -r '${mrLsMemberNamesJq}' \ | while IFS= read -r member; do [ -n "$member" ] || continue case ",''${MEGAREPO_SKIP_MEMBERS:-}," in diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 46499e5c8..b62e1c4cc 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -197,6 +197,7 @@ let # ordered line stream that the previous bash implementation produced. NODE_MODULES_HELPER_MODE="projection-hash" \ PNPM_ROOT_MODULES_YAML="node_modules/.modules.yaml" \ + PNPM_GVS_LINKS_DIR="$(resolve_gvs_links_dir)" \ NODE_MODULES_DIRS="$(printf '%s\n' node_modules ${nodeModulesPaths})" \ ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} } diff --git a/nix/devenv-modules/tasks/shared/setup.nix b/nix/devenv-modules/tasks/shared/setup.nix index e0954d1b4..acbdcd06e 100644 --- a/nix/devenv-modules/tasks/shared/setup.nix +++ b/nix/devenv-modules/tasks/shared/setup.nix @@ -387,7 +387,8 @@ in # Required tasks are hard dependencies; optional tasks use @completed so # failures don't block shell entry. "devenv:enterShell" = { - after = setupRequiredTasks + after = + setupRequiredTasks ++ (map (t: "${t}@completed") setupOptionalTasks) ++ lib.optionals (setupTasks != [ ]) [ "${setupRecordCacheTaskName}@completed" ]; }; diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index 4dcd581b3..0851292b7 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -24,7 +24,7 @@ extract_task_script() { local module_args="${4:-packages = [ ];}" local task_name="${5:-pnpm:install}" - nix eval --impure --raw --expr " + nix-instantiate --eval --strict --json --expr " let flake = builtins.getFlake (toString $ROOT); pkgs = import flake.inputs.nixpkgs { system = builtins.currentSystem; }; @@ -41,7 +41,7 @@ extract_task_script() { config = { devenv.root = \"$workspace_root\"; }; }; in (builtins.getAttr \"${task_name}\" module.tasks).${attr} - " > "$output_path" + " | jq -r . > "$output_path" chmod +x "$output_path" } @@ -52,7 +52,7 @@ extract_shared_task_script() { local package_name="$4" local output_path="$5" - nix eval --impure --raw --expr " + nix-instantiate --eval --strict --json --expr " let flake = builtins.getFlake (toString $ROOT); pkgs = import flake.inputs.nixpkgs { system = builtins.currentSystem; }; @@ -83,7 +83,7 @@ extract_shared_task_script() { ]; }; in evaluated.config.tasks.\"${task_name}\".exec - " > "$output_path" + " | jq -r . > "$output_path" chmod +x "$output_path" } @@ -94,7 +94,12 @@ rewrite_unrealized_tool_paths() { # the referenced helper packages. Patch the generated absolute store paths to # temp-local shims so the test only exercises task behavior, not derivation # realisation. - perl -0pi -e 's#/nix/store/[^"\s]*/bin/flock#'"$tmpdir"'/bin/flock#g; s#/nix/store/[^"\s]*/bin/node#node#g' "$script_path" + perl -0pi -e ' + s#/nix/store/[^"\s]*/bin/flock#'"$tmpdir"'/bin/flock#g; + s#/nix/store/[^"\s]*/bin/node#node#g; + s#/nix/store/[^"\s]*-pnpm-task-helpers\.sh#'"$ROOT"'/nix/devenv-modules/tasks/shared/pnpm-task-helpers.sh#g; + s#/nix/store/[^"\s]*-check-node-modules-projection-health\.cjs#'"$ROOT"'/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs#g; + ' "$script_path" } echo "Running pnpm task smoke test..." @@ -373,6 +378,7 @@ echo "Test 9: outer cache hit misses when a projected symlink disappears" set -e unset DEVENV_SETUP_OUTER_CACHE_HIT assert_exit_code 1 "$exit_code" "outer-hit status should miss when a projected symlink disappears" + bash "$tmpdir/pnpm-install.exec.sh" ) echo "Test 10: status still hits when PNPM_HOME changes but store-dir stays shared" diff --git a/nix/devenv-modules/tasks/shared/ts.nix b/nix/devenv-modules/tasks/shared/ts.nix index 49a2f1e88..149f62234 100644 --- a/nix/devenv-modules/tasks/shared/ts.nix +++ b/nix/devenv-modules/tasks/shared/ts.nix @@ -54,64 +54,64 @@ let "pnpm:install" ]; emitTsconfigHelper = '' - generate_emit_tsconfig() { - local source_tsconfig="$1" - local target_tsconfig="$2" + generate_emit_tsconfig() { + local source_tsconfig="$1" + local target_tsconfig="$2" - # `tsc --build --dry --noCheck` still treats `noEmit` references as emit - # work, which made `ts:emit` look perpetually stale. Build a filtered - # graph just for this task instead of mutating the checked-in config. - ${pkgs.nodejs}/bin/node - "$source_tsconfig" "$target_tsconfig" <<'NODE' -const fs = require('node:fs') -const path = require('node:path') + # `tsc --build --dry --noCheck` still treats `noEmit` references as emit + # work, which made `ts:emit` look perpetually stale. Build a filtered + # graph just for this task instead of mutating the checked-in config. + ${pkgs.nodejs}/bin/node - "$source_tsconfig" "$target_tsconfig" <<'NODE' + const fs = require('node:fs') + const path = require('node:path') -const [sourceTsconfig, targetTsconfig] = process.argv.slice(2) + const [sourceTsconfig, targetTsconfig] = process.argv.slice(2) -const loadTypescript = () => { - try { - return require(require.resolve('typescript', { paths: [path.dirname(sourceTsconfig), process.cwd()] })) - } catch (error) { - throw new Error( - 'Unable to resolve TypeScript while preparing ts:emit: ' + - String(error?.message ?? error) - ) - } -} + const loadTypescript = () => { + try { + return require(require.resolve('typescript', { paths: [path.dirname(sourceTsconfig), process.cwd()] })) + } catch (error) { + throw new Error( + 'Unable to resolve TypeScript while preparing ts:emit: ' + + String(error?.message ?? error) + ) + } + } -const typescript = loadTypescript() + const typescript = loadTypescript() -const readTsconfig = (filePath) => { - const parsed = typescript.readConfigFile(filePath, (path) => fs.readFileSync(path, 'utf8')) - if (parsed.error) { - const message = typeof parsed.error.messageText === 'string' - ? parsed.error.messageText - : JSON.stringify(parsed.error.messageText) - throw new Error('Failed to parse ' + filePath + ': ' + message) - } - return parsed.config -} + const readTsconfig = (filePath) => { + const parsed = typescript.readConfigFile(filePath, (path) => fs.readFileSync(path, 'utf8')) + if (parsed.error) { + const message = typeof parsed.error.messageText === 'string' + ? parsed.error.messageText + : JSON.stringify(parsed.error.messageText) + throw new Error('Failed to parse ' + filePath + ': ' + message) + } + return parsed.config + } -const resolveReferenceTsconfig = (referencePath) => { - const resolvedPath = path.resolve(baseDir, referencePath) - return path.extname(resolvedPath) ? resolvedPath : path.join(resolvedPath, 'tsconfig.json') -} + const resolveReferenceTsconfig = (referencePath) => { + const resolvedPath = path.resolve(baseDir, referencePath) + return path.extname(resolvedPath) ? resolvedPath : path.join(resolvedPath, 'tsconfig.json') + } -const rootConfig = readTsconfig(sourceTsconfig) -const baseDir = path.dirname(sourceTsconfig) + const rootConfig = readTsconfig(sourceTsconfig) + const baseDir = path.dirname(sourceTsconfig) -rootConfig.references = (rootConfig.references ?? []).filter((reference) => { - const refTsconfig = resolveReferenceTsconfig(reference.path) - if (!fs.existsSync(refTsconfig)) { - return true - } + rootConfig.references = (rootConfig.references ?? []).filter((reference) => { + const refTsconfig = resolveReferenceTsconfig(reference.path) + if (!fs.existsSync(refTsconfig)) { + return true + } - const refConfig = readTsconfig(refTsconfig) - return refConfig.compilerOptions?.noEmit !== true -}) + const refConfig = readTsconfig(refTsconfig) + return refConfig.compilerOptions?.noEmit !== true + }) -fs.writeFileSync(targetTsconfig, JSON.stringify(rootConfig)) -NODE - } + fs.writeFileSync(targetTsconfig, JSON.stringify(rootConfig)) + NODE + } ''; # Script that runs tsc with --extendedDiagnostics --verbose,