diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8350205f3..6e53ab2c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2078,6 +2078,498 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" + devenv-perf: + runs-on: + [namespace-profile-linux-x86-64, 'namespace-features:github.run-id=${{ github.run_id }}'] + defaults: + run: + shell: bash + env: + FORCE_SETUP: '1' + CI: 'true' + GITHUB_TOKEN: ${{ github.token }} + ARTIFACT_DIR: tmp/devenv-perf-ci + OTEL_SERVICE_NAME: devenv-perf-ci + DEVENV_PERF_REGRESSION_MODE: warn + RUNNER_CLASS: 'namespace-profile-linux-x86-64,namespace-features:github.run-id=${{ github.run_id }}' + steps: + - uses: actions/checkout@v6 + - name: Install Nix + uses: DeterminateSystems/determinate-nix-action@v3 + with: + extra-conf: | + experimental-features = nix-command flakes + accept-flake-config = true + extra-substituters = https://devenv.cachix.org + extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= + access-tokens = github.com=${{ github.token }} + summarize: true + - name: Enable Cachix cache + uses: cachix/cachix-action@v17 + with: + name: overeng-effect-utils + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Use pinned devenv from lock + run: | + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then + echo '::error::devenv.lock missing .nodes.devenv.locked.rev' + exit 1 + fi + echo "DEVENV_REV=$DEVENV_REV" >> "$GITHUB_ENV" + echo "Pinned devenv rev: $DEVENV_REV" + shell: bash + - name: Isolate pnpm state + shell: bash + run: | + echo "PNPM_STORE_DIR=${{ runner.temp }}/pnpm-store/${{ github.job }}" >> "$GITHUB_ENV" + echo "PNPM_HOME=${{ github.workspace }}/.pnpm-home" >> "$GITHUB_ENV" + - id: restore-pnpm-state + name: Restore pnpm state + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.pnpm-home + ${{ runner.temp }}/pnpm-store/${{ github.job }} + key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" + - name: Resolve devenv + run: | + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then + echo '::error::devenv.lock missing .nodes.devenv.locked.rev' + exit 1 + fi + + resolve_devenv() { + nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" + } + + # Temporary: capture diagnostics dir for #272 root-cause analysis. + DIAG_ROOT="${RUNNER_TEMP:-/tmp}/nix-store-diagnostics-${GITHUB_JOB:-job}-${RUNNER_OS:-unknown}-${GITHUB_RUN_ATTEMPT:-0}" + mkdir -p "$DIAG_ROOT" + echo "NIX_STORE_DIAGNOSTICS_DIR=$DIAG_ROOT" >> "$GITHUB_ENV" + + { + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "github_job=${GITHUB_JOB:-unknown}" + echo "github_run_id=${GITHUB_RUN_ID:-unknown}" + echo "nix_user_conf_files=${NIX_USER_CONF_FILES:-}" + nix --version || true + } > "$DIAG_ROOT/environment.txt" 2>&1 + + if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv.log" >&2)); then + echo "::error::resolve_devenv failed. Last 30 lines of log:" + tail -30 "$DIAG_ROOT/resolve-devenv.log" || true + exit 1 + fi + DEVENV_BIN="$DEVENV_OUT/bin/devenv" + + # Fast validity check on the devenv store path (~1-2s vs ~25s for devenv info). + if ! nix-store --check-validity "$DEVENV_OUT" 2>/dev/null; then + echo "::warning::devenv store path invalid, repairing targeted path..." + nix-store --repair-path "$DEVENV_OUT" > "$DIAG_ROOT/nix-store-verify-repair.log" 2>&1 || true + rm -rf "${XDG_CACHE_HOME:-$HOME/.cache}"/nix/eval-cache-* ~/.cache/nix/eval-cache-* + if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv-post-repair.log" >&2)); then + echo "::error::resolve_devenv failed after repair. Last 30 lines of log:" + tail -30 "$DIAG_ROOT/resolve-devenv-post-repair.log" || true + exit 1 + fi + DEVENV_BIN="$DEVENV_OUT/bin/devenv" + fi + + echo "DEVENV_BIN=$DEVENV_BIN" >> "$GITHUB_ENV" + "$DEVENV_BIN" version | tee "$DIAG_ROOT/devenv-version.txt" + shell: bash + - name: Evict cached pnpm deps for oxlint-npm + shell: bash + run: | + targetRef='.#oxlint-npm' + entriesJson=$(mktemp) + if nix eval --json "$targetRef.passthru.depsBuildEntries" >"$entriesJson" 2>/dev/null; then + while IFS=$'\t' read -r attrName drv; do + [ -n "$drv" ] || continue + while IFS= read -r outPath; do + [ -n "$outPath" ] || continue + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "evicting cached: $(basename "$outPath")" + if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then + echo "::error::failed to evict cached pnpm-deps output: $outPath" + exit 1 + fi + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "::error::cached pnpm-deps output still present after eviction: $outPath" + exit 1 + fi + fi + done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) + done < <(jq -r '.[] | [.attrName, (.drvPath // "")] | @tsv' "$entriesJson") + else + topDrv=$(nix path-info --derivation "$targetRef" 2>/dev/null || true) + if [ -n "$topDrv" ]; then + while IFS= read -r drv; do + [ -n "$drv" ] || continue + attrName="" + while IFS= read -r outPath; do + [ -n "$outPath" ] || continue + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "evicting cached: $(basename "$outPath")" + if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then + echo "::error::failed to evict cached pnpm-deps output: $outPath" + exit 1 + fi + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "::error::cached pnpm-deps output still present after eviction: $outPath" + exit 1 + fi + fi + done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) + done < <(nix-store -qR "$topDrv" 2>/dev/null | grep "pnpm-deps-[a-z0-9-]*-v[0-9].*\.drv$" || true) + fi + fi + rm -f "$entriesJson" + - name: Force diagnostics failure (debug) + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} + shell: bash + run: | + diag_dir="${NIX_STORE_DIAGNOSTICS_DIR:-${RUNNER_TEMP:-/tmp}/nix-store-diagnostics-missing}" + mkdir -p "$diag_dir" + cat > "$diag_dir/synthetic-signature.log" <<'EOF' + Failed to convert config.cachix to JSON + ... while evaluating the option `cachix.package` + error: path '/nix/store/synthetic-invalid-path' is not valid + EOF + echo "::warning::Intentional failure for diagnostics validation (#272)" + exit 1 + - name: Benchmark devenv surfaces + shell: bash + run: | + set -euo pipefail + + mkdir -p "$ARTIFACT_DIR/traces" + + { + printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'repository=%s\n' "${GITHUB_REPOSITORY:-unknown}" + printf 'ref=%s\n' "${GITHUB_REF:-unknown}" + printf 'sha=%s\n' "${GITHUB_SHA:-unknown}" + printf 'runner_name=%s\n' "${RUNNER_NAME:-unknown}" + printf 'runner_os=%s\n' "${RUNNER_OS:-unknown}" + printf 'runner_arch=%s\n' "${RUNNER_ARCH:-unknown}" + printf 'devenv_rev=%s\n' "${DEVENV_REV:-unknown}" + printf 'otel_service_name=%s\n' "${OTEL_SERVICE_NAME:-unknown}" + df -h / /nix 2>/dev/null || df -h / + ps -eo pid,ppid,stat,etime,pcpu,pmem,comm,args 2>/dev/null \ + | grep -E 'devenv direnv-export|nix-daemon|nix build|nix flake|github-runner' \ + | grep -v grep || true + } >"$ARTIFACT_DIR/host-context.txt" + + printf '[' >"$ARTIFACT_DIR/timings.json" + first=1 + + json_append_timing() { + local name="$1" + local status="$2" + local duration_ms="$3" + local stdout="$4" + local stderr="$5" + local trace="$6" + + if [ "$first" -eq 0 ]; then + printf ',' >>"$ARTIFACT_DIR/timings.json" + fi + first=0 + + jq -cn \ + --arg name "$name" \ + --argjson status "$status" \ + --argjson durationMs "$duration_ms" \ + --arg stdout "$stdout" \ + --arg stderr "$stderr" \ + --arg trace "$trace" \ + '{name:$name,status:$status,durationMs:$durationMs,stdout:$stdout,stderr:$stderr,trace:(if $trace == "" then null else $trace end)}' \ + >>"$ARTIFACT_DIR/timings.json" + } + + measure() { + local name="$1" + local trace_file="$2" + shift 2 + case "$trace_file" in + '$ARTIFACT_DIR'*) trace_file="${ARTIFACT_DIR}${trace_file#'$ARTIFACT_DIR'}" ;; + esac + local stdout="$ARTIFACT_DIR/$name.stdout" + local stderr="$ARTIFACT_DIR/$name.stderr" + local started ended status duration_ms + + mkdir -p "$(dirname "$trace_file")" + started="$(date +%s%3N)" + set +e + expanded=() + for arg in "$@"; do + case "$arg" in + '$DEVENV_BIN') expanded+=("${DEVENV_BIN:?DEVENV_BIN not set}") ;; + '$ARTIFACT_DIR'*) expanded+=("${ARTIFACT_DIR}${arg#'$ARTIFACT_DIR'}") ;; + 'json:file:$trace_file') expanded+=("json:file:$trace_file") ;; + '$trace_file') expanded+=("file:$trace_file") ;; + *) expanded+=("$arg") ;; + esac + done + "${expanded[@]}" >"$stdout" 2>"$stderr" + status=$? + set -e + ended="$(date +%s%3N)" + duration_ms=$((ended - started)) + + json_append_timing "$name" "$status" "$duration_ms" "$stdout" "$stderr" "$trace_file" + + if [ "$status" -ne 0 ]; then + echo "::error::$name failed after ${duration_ms}ms; stderr tail follows" + tail -80 "$stderr" || true + return "$status" + fi + } + + measure 'shell_eval_traced' '$ARTIFACT_DIR/traces/shell_eval_traced.json' '$DEVENV_BIN' '--trace-to' 'json:file:$trace_file' 'shell' '--no-reload' '--' 'true' + measure 'shell_eval_warm' '' '$DEVENV_BIN' 'shell' '--no-reload' '--' 'true' + measure 'tasks_list' '' '$DEVENV_BIN' 'tasks' 'list' + measure 'processes_help' '' '$DEVENV_BIN' 'processes' '--help' + measure 'task_pnpm_install' '' '$DEVENV_BIN' 'tasks' 'run' 'pnpm:install' '--mode' 'before' '--no-tui' '--show-output' + measure 'task_genie_run' '' '$DEVENV_BIN' 'tasks' 'run' 'genie:run' '--mode' 'before' '--no-tui' '--show-output' + measure 'task_check_quick' '' '$DEVENV_BIN' 'tasks' 'run' 'check:quick' '--mode' 'before' '--no-tui' '--show-output' + + printf ']\n' >>"$ARTIFACT_DIR/timings.json" + + jq . "$ARTIFACT_DIR/timings.json" >"$ARTIFACT_DIR/timings.pretty.json" + jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --arg schemaVersion "1" \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${GITHUB_REPOSITORY:-unknown}" \ + --arg ref "${GITHUB_REF:-unknown}" \ + --arg sha "${GITHUB_SHA:-unknown}" \ + --arg runnerName "${RUNNER_NAME:-unknown}" \ + --arg runnerOs "${RUNNER_OS:-unknown}" \ + --arg runnerArch "${RUNNER_ARCH:-unknown}" \ + --arg devenvRev "${DEVENV_REV:-unknown}" \ + --arg otelServiceName "${OTEL_SERVICE_NAME:-unknown}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + repository: $repository, + ref: $ref, + sha: $sha, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch }, + devenv: { rev: $devenvRev }, + otel: { serviceName: $otelServiceName }, + checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) + }' >"$ARTIFACT_DIR/summary.json" + + jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --argjson schemaVersion 1 \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${GITHUB_REPOSITORY:-unknown}" \ + --arg branchKind "${GITHUB_EVENT_NAME:-unknown}" \ + --arg ref "${GITHUB_REF:-unknown}" \ + --arg headSha "${GITHUB_SHA:-unknown}" \ + --arg baseSha "${GITHUB_BASE_SHA:-}" \ + --arg runnerName "${RUNNER_NAME:-unknown}" \ + --arg runnerOs "${RUNNER_OS:-unknown}" \ + --arg runnerArch "${RUNNER_ARCH:-unknown}" \ + --arg runnerClass "${RUNNER_CLASS:-unknown}" \ + --arg githubRunId "${GITHUB_RUN_ID:-unknown}" \ + --arg githubRunAttempt "${GITHUB_RUN_ATTEMPT:-unknown}" \ + --arg githubJob "${GITHUB_JOB:-unknown}" \ + --arg taskId "${CROSSTASK_TASK_ID:-}" \ + --arg taskAttemptId "${CROSSTASK_ATTEMPT_ID:-}" \ + --arg traceId "${TRACE_ID:-}" \ + --arg devenvRev "${DEVENV_REV:-unknown}" \ + --arg otelServiceName "${OTEL_SERVICE_NAME:-unknown}" \ + --arg targetSystem "${DEVENV_SYSTEM:-${RUNNER_OS:-unknown}}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + producer: { name: "effect-utils-ci-measurement", version: 1 }, + subject: { + repo: $repository, + branchKind: (if $branchKind == "" then "unknown" else $branchKind end), + ref: $ref, + headSha: $headSha, + baseSha: $baseSha + }, + execution: { + provider: (if ($githubRunId != "" and $githubRunId != "unknown") then "github-actions" else "local" end), + workflow: "CI", + job: $githubJob, + runId: $githubRunId, + runAttempt: $githubRunAttempt, + taskId: $taskId, + attemptId: $taskAttemptId, + traceId: $traceId, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch, class: $runnerClass } + }, + target: { kind: "devenv", name: "dev-shell", system: $targetSystem }, + observations: ( + $timings[0] + | map({ + name: ("devenv." + .name + ".duration"), + unit: "seconds", + value: (.durationMs / 1000), + dimensions: { + probe: .name, + status: .status, + devenvRev: $devenvRev, + otelServiceName: $otelServiceName + } + }) + ), + artifacts: [ + { name: "host-context", path: "host-context.txt", contentType: "text/plain" }, + { name: "timings", path: "timings.json", contentType: "application/json" }, + { name: "summary", path: "summary.json", contentType: "application/json" }, + { name: "shell-eval-trace", path: "traces/shell_eval_traced.json", contentType: "application/json" } + ], + details: { + stdoutStderrByProbe: ( + $timings[0] + | map({ key: .name, value: { stdout: .stdout, stderr: .stderr, trace: .trace } }) + | from_entries + ) + } + }' >"$ARTIFACT_DIR/measurements.json" + + compare_baseline() { + local baseline_path="${DEVENV_PERF_BASELINE_SUMMARY:-$ARTIFACT_DIR/baseline/summary.json}" + local mode="${DEVENV_PERF_REGRESSION_MODE:-warn}" + + if [ "$mode" = "off" ]; then + jq -n --argjson schemaVersion 1 --arg status skipped --arg mode "$mode" '{schemaVersion:$schemaVersion, status:$status, mode:$mode, checks:{}}' >"$ARTIFACT_DIR/perf-comparison.json" + return 0 + fi + + if [ ! -f "$baseline_path" ]; then + jq -n \ + --argjson schemaVersion 1 \ + --arg status baseline_missing \ + --arg mode "$mode" \ + --arg baseline "$baseline_path" \ + '{schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baseline, checks:{}}' \ + >"$ARTIFACT_DIR/perf-comparison.json" + echo "::notice::devenv perf baseline not found at $baseline_path; recorded current measurements only" + return 0 + fi + + jq -n \ + --slurpfile current "$ARTIFACT_DIR/summary.json" \ + --slurpfile baseline "$baseline_path" \ + --argjson schemaVersion 1 \ + --arg mode "$mode" \ + --arg baselinePath "$baseline_path" \ + ' + def budget($name): + if $name == "shell_eval_traced" then + {warnRatio:1.25, failRatio:1.5, warnMs:1500, failMs:3000} + elif $name == "shell_eval_warm" then + {warnRatio:1.5, failRatio:2.0, warnMs:500, failMs:1000} + elif $name == "tasks_list" or $name == "processes_help" then + {warnRatio:2.0, failRatio:3.0, warnMs:250, failMs:1000} + else + {warnRatio:1.5, failRatio:2.0, warnMs:1000, failMs:3000} + end; + def classify($name; $current; $baseline): + budget($name) as $b + | ($current - $baseline) as $delta + | (if $baseline > 0 then ($current / $baseline) else null end) as $ratio + | if $baseline <= 0 then "unknown" + elif ($delta > $b.failMs and $current > ($baseline * $b.failRatio)) then "fail" + elif ($delta > $b.warnMs and $current > ($baseline * $b.warnRatio)) then "warn" + else "pass" + end as $status + | {status:$status, currentMs:$current, baselineMs:$baseline, deltaMs:$delta, ratio:$ratio, budget:$b}; + ($current[0].checks // {}) as $currentChecks + | ($baseline[0].checks // {}) as $baselineChecks + | ( + $currentChecks + | to_entries + | map( + .key as $name + | .value as $current + | ($baselineChecks[$name] // null) as $base + | { + key: $name, + value: + if $base == null then + {status:"missing_baseline", currentMs:$current.durationMs} + elif ($current.status != 0) then + {status:"current_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + elif ($base.status != 0) then + {status:"baseline_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + else + classify($name; $current.durationMs; $base.durationMs) + end + } + ) + | from_entries + ) as $checks + | ( + if any($checks[]; .status == "fail") then "fail" + elif any($checks[]; .status == "warn") then "warn" + elif any($checks[]; .status == "missing_baseline") then "partial" + else "pass" + end + ) as $status + | {schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baselinePath, checks:$checks} + ' >"$ARTIFACT_DIR/perf-comparison.json" + + local status + status="$(jq -r '.status' "$ARTIFACT_DIR/perf-comparison.json")" + case "$status:$mode" in + fail:fail) + echo "::error::devenv perf regression detected" + jq . "$ARTIFACT_DIR/perf-comparison.json" + return 1 + ;; + fail:*|warn:*) + echo "::warning::devenv perf regression threshold exceeded" + jq . "$ARTIFACT_DIR/perf-comparison.json" + ;; + esac + } + + compare_baseline + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Devenv perf" + echo "" + echo "| Probe | Status | Duration |" + echo "| --- | ---: | ---: |" + jq -r '.[] | "| \(.name) | \(.status) | \(.durationMs) ms |"' "$ARTIFACT_DIR/timings.json" + echo "" + echo "- Artifact directory: \`$ARTIFACT_DIR\`" + echo "- OTEL service: \`${OTEL_SERVICE_NAME:-unknown}\`" + echo "" + echo "#### Regression comparison" + echo "" + if [ -f "$ARTIFACT_DIR/perf-comparison.json" ]; then + jq -r '["- Status: " + .status, "- Mode: " + .mode, "- Baseline: " + (.baseline // "none")] | .[]' "$ARTIFACT_DIR/perf-comparison.json" + fi + } >>"$GITHUB_STEP_SUMMARY" + fi + + cat "$ARTIFACT_DIR/timings.pretty.json" + + - name: Upload devenv perf artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: 'devenv-perf-${{ github.job }}-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: tmp/devenv-perf-ci + if-no-files-found: error + retention-days: 30 test-integration-notion: runs-on: [namespace-profile-linux-x86-64, 'namespace-features:github.run-id=${{ github.run_id }}'] diff --git a/.github/workflows/ci.yml.genie.ts b/.github/workflows/ci.yml.genie.ts index c4faf73f1..fca8ec180 100644 --- a/.github/workflows/ci.yml.genie.ts +++ b/.github/workflows/ci.yml.genie.ts @@ -14,6 +14,7 @@ import { savePnpmStateStep, standardCIEnv, ciWorkflow, + devenvPerfJob, namespaceRunner, validateColdPnpmDepsStep, nixDiagnosticsArtifactStep, @@ -251,6 +252,14 @@ const NETLIFY_SITE = 'overeng-utils' // Non-required jobs (separate from CIJobName — not required status checks) const extraJobs: Record = { + 'devenv-perf': devenvPerfJob({ + runsOn: namespaceRunner({ + profile: 'namespace-profile-linux-x86-64', + runId: '${{ github.run_id }}', + }), + setupSteps: baseSteps, + taskProbes: ['pnpm:install', 'genie:run', 'check:quick'], + }), /** Integration tests for Notion API (requires NOTION_TOKEN secret) */ 'test-integration-notion': { 'runs-on': namespaceRunner({ diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index bbaf243de..2d4feb885 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -348,6 +348,589 @@ export const runDevenvTasksBefore = (...args: [string, ...string[]]) => label: `devenv tasks run ${args.join(' ')} --mode before`, }) +export type DevenvPerfProbe = { + readonly name: string + readonly command: readonly [string, ...string[]] + readonly traceOutput?: string +} + +export type CiMeasurementObservation = { + readonly name: string + readonly unit: string + readonly value: number + readonly dimensions?: Record +} + +export const ciMeasurementMetrics = { + devenvProbeDuration: 'devenv..duration', + nixClosureNarSize: 'nix.closure.nar_size', + nixClosurePathCount: 'nix.closure.path_count', + nixClosureBucketNarSize: 'nix.closure.bucket.nar_size', +} as const + +export type NixClosureMeasurementBucket = { + readonly name: string + readonly pathRegex: string +} + +export type NixClosureMeasurementStepOptions = { + readonly installable: string + readonly targetName?: string + readonly targetSystem?: string + readonly artifactDir?: string + readonly artifactFile?: string + readonly buckets?: readonly NixClosureMeasurementBucket[] +} + +type DevenvPerfSetupStep = GitHubWorkflowArgs['jobs'][string]['steps'][number] + +export type DevenvPerfJobOptions = { + readonly runsOn?: readonly string[] + readonly artifactDir?: string + readonly artifactName?: string + readonly setupSteps?: readonly DevenvPerfSetupStep[] + readonly env?: Record + readonly taskProbes?: readonly string[] + readonly probes?: readonly DevenvPerfProbe[] + readonly retentionDays?: number + readonly regressionMode?: 'off' | 'warn' | 'fail' +} + +const devenvPerfProbeLine = (probe: DevenvPerfProbe) => { + const args = probe.command.map(shellSingleQuote).join(' ') + const trace = probe.traceOutput ?? '' + return `measure ${shellSingleQuote(probe.name)} ${shellSingleQuote(trace)} ${args}` +} + +const defaultDevenvPerfTaskProbe = (task: string): DevenvPerfProbe => ({ + name: `task_${task.replaceAll(':', '_')}`, + command: ['$DEVENV_BIN', 'tasks', 'run', task, '--mode', 'before', '--no-tui', '--show-output'], +}) + +const renderDevenvPerfScript = ( + opts: Required>, +) => { + const probes: readonly DevenvPerfProbe[] = [ + { + name: 'shell_eval_traced', + command: [ + '$DEVENV_BIN', + '--trace-to', + 'json:file:$trace_file', + 'shell', + '--no-reload', + '--', + 'true', + ], + traceOutput: '$ARTIFACT_DIR/traces/shell_eval_traced.json', + }, + { name: 'shell_eval_warm', command: ['$DEVENV_BIN', 'shell', '--no-reload', '--', 'true'] }, + { name: 'tasks_list', command: ['$DEVENV_BIN', 'tasks', 'list'] }, + { name: 'processes_help', command: ['$DEVENV_BIN', 'processes', '--help'] }, + ...opts.taskProbes.map(defaultDevenvPerfTaskProbe), + ...opts.probes, + ] + + return String.raw`set -euo pipefail + +mkdir -p "$ARTIFACT_DIR/traces" + +{ + printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'repository=%s\n' "${dollar}{GITHUB_REPOSITORY:-unknown}" + printf 'ref=%s\n' "${dollar}{GITHUB_REF:-unknown}" + printf 'sha=%s\n' "${dollar}{GITHUB_SHA:-unknown}" + printf 'runner_name=%s\n' "${dollar}{RUNNER_NAME:-unknown}" + printf 'runner_os=%s\n' "${dollar}{RUNNER_OS:-unknown}" + printf 'runner_arch=%s\n' "${dollar}{RUNNER_ARCH:-unknown}" + printf 'devenv_rev=%s\n' "${dollar}{DEVENV_REV:-unknown}" + printf 'otel_service_name=%s\n' "${dollar}{OTEL_SERVICE_NAME:-unknown}" + df -h / /nix 2>/dev/null || df -h / + ps -eo pid,ppid,stat,etime,pcpu,pmem,comm,args 2>/dev/null \ + | grep -E 'devenv direnv-export|nix-daemon|nix build|nix flake|github-runner' \ + | grep -v grep || true +} >"$ARTIFACT_DIR/host-context.txt" + +printf '[' >"$ARTIFACT_DIR/timings.json" +first=1 + +json_append_timing() { + local name="$1" + local status="$2" + local duration_ms="$3" + local stdout="$4" + local stderr="$5" + local trace="$6" + + if [ "$first" -eq 0 ]; then + printf ',' >>"$ARTIFACT_DIR/timings.json" + fi + first=0 + + jq -cn \ + --arg name "$name" \ + --argjson status "$status" \ + --argjson durationMs "$duration_ms" \ + --arg stdout "$stdout" \ + --arg stderr "$stderr" \ + --arg trace "$trace" \ + '{name:$name,status:$status,durationMs:$durationMs,stdout:$stdout,stderr:$stderr,trace:(if $trace == "" then null else $trace end)}' \ + >>"$ARTIFACT_DIR/timings.json" +} + +measure() { + local name="$1" + local trace_file="$2" + shift 2 + case "$trace_file" in + '$ARTIFACT_DIR'*) trace_file="${dollar}{ARTIFACT_DIR}${dollar}{trace_file#'$ARTIFACT_DIR'}" ;; + esac + local stdout="$ARTIFACT_DIR/$name.stdout" + local stderr="$ARTIFACT_DIR/$name.stderr" + local started ended status duration_ms + + mkdir -p "$(dirname "$trace_file")" + started="$(date +%s%3N)" + set +e + expanded=() + for arg in "$@"; do + case "$arg" in + '$DEVENV_BIN') expanded+=("${dollar}{DEVENV_BIN:?DEVENV_BIN not set}") ;; + '$ARTIFACT_DIR'*) expanded+=("${dollar}{ARTIFACT_DIR}${dollar}{arg#'$ARTIFACT_DIR'}") ;; + 'json:file:$trace_file') expanded+=("json:file:$trace_file") ;; + '$trace_file') expanded+=("file:$trace_file") ;; + *) expanded+=("$arg") ;; + esac + done + "${dollar}{expanded[@]}" >"$stdout" 2>"$stderr" + status=$? + set -e + ended="$(date +%s%3N)" + duration_ms=$((ended - started)) + + json_append_timing "$name" "$status" "$duration_ms" "$stdout" "$stderr" "$trace_file" + + if [ "$status" -ne 0 ]; then + echo "::error::$name failed after ${dollar}{duration_ms}ms; stderr tail follows" + tail -80 "$stderr" || true + return "$status" + fi +} + +${probes.map(devenvPerfProbeLine).join('\n')} + +printf ']\n' >>"$ARTIFACT_DIR/timings.json" + +jq . "$ARTIFACT_DIR/timings.json" >"$ARTIFACT_DIR/timings.pretty.json" +jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --arg schemaVersion "1" \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${dollar}{GITHUB_REPOSITORY:-unknown}" \ + --arg ref "${dollar}{GITHUB_REF:-unknown}" \ + --arg sha "${dollar}{GITHUB_SHA:-unknown}" \ + --arg runnerName "${dollar}{RUNNER_NAME:-unknown}" \ + --arg runnerOs "${dollar}{RUNNER_OS:-unknown}" \ + --arg runnerArch "${dollar}{RUNNER_ARCH:-unknown}" \ + --arg devenvRev "${dollar}{DEVENV_REV:-unknown}" \ + --arg otelServiceName "${dollar}{OTEL_SERVICE_NAME:-unknown}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + repository: $repository, + ref: $ref, + sha: $sha, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch }, + devenv: { rev: $devenvRev }, + otel: { serviceName: $otelServiceName }, + checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) + }' >"$ARTIFACT_DIR/summary.json" + +jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --argjson schemaVersion 1 \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${dollar}{GITHUB_REPOSITORY:-unknown}" \ + --arg branchKind "${dollar}{GITHUB_EVENT_NAME:-unknown}" \ + --arg ref "${dollar}{GITHUB_REF:-unknown}" \ + --arg headSha "${dollar}{GITHUB_SHA:-unknown}" \ + --arg baseSha "${dollar}{GITHUB_BASE_SHA:-}" \ + --arg runnerName "${dollar}{RUNNER_NAME:-unknown}" \ + --arg runnerOs "${dollar}{RUNNER_OS:-unknown}" \ + --arg runnerArch "${dollar}{RUNNER_ARCH:-unknown}" \ + --arg runnerClass "${dollar}{RUNNER_CLASS:-unknown}" \ + --arg githubRunId "${dollar}{GITHUB_RUN_ID:-unknown}" \ + --arg githubRunAttempt "${dollar}{GITHUB_RUN_ATTEMPT:-unknown}" \ + --arg githubJob "${dollar}{GITHUB_JOB:-unknown}" \ + --arg taskId "${dollar}{CROSSTASK_TASK_ID:-}" \ + --arg taskAttemptId "${dollar}{CROSSTASK_ATTEMPT_ID:-}" \ + --arg traceId "${dollar}{TRACE_ID:-}" \ + --arg devenvRev "${dollar}{DEVENV_REV:-unknown}" \ + --arg otelServiceName "${dollar}{OTEL_SERVICE_NAME:-unknown}" \ + --arg targetSystem "${dollar}{DEVENV_SYSTEM:-${dollar}{RUNNER_OS:-unknown}}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + producer: { name: "effect-utils-ci-measurement", version: 1 }, + subject: { + repo: $repository, + branchKind: (if $branchKind == "" then "unknown" else $branchKind end), + ref: $ref, + headSha: $headSha, + baseSha: $baseSha + }, + execution: { + provider: (if ($githubRunId != "" and $githubRunId != "unknown") then "github-actions" else "local" end), + workflow: "CI", + job: $githubJob, + runId: $githubRunId, + runAttempt: $githubRunAttempt, + taskId: $taskId, + attemptId: $taskAttemptId, + traceId: $traceId, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch, class: $runnerClass } + }, + target: { kind: "devenv", name: "dev-shell", system: $targetSystem }, + observations: ( + $timings[0] + | map({ + name: ("devenv." + .name + ".duration"), + unit: "seconds", + value: (.durationMs / 1000), + dimensions: { + probe: .name, + status: .status, + devenvRev: $devenvRev, + otelServiceName: $otelServiceName + } + }) + ), + artifacts: [ + { name: "host-context", path: "host-context.txt", contentType: "text/plain" }, + { name: "timings", path: "timings.json", contentType: "application/json" }, + { name: "summary", path: "summary.json", contentType: "application/json" }, + { name: "shell-eval-trace", path: "traces/shell_eval_traced.json", contentType: "application/json" } + ], + details: { + stdoutStderrByProbe: ( + $timings[0] + | map({ key: .name, value: { stdout: .stdout, stderr: .stderr, trace: .trace } }) + | from_entries + ) + } + }' >"$ARTIFACT_DIR/measurements.json" + +compare_baseline() { + local baseline_path="${dollar}{DEVENV_PERF_BASELINE_SUMMARY:-$ARTIFACT_DIR/baseline/summary.json}" + local mode="${dollar}{DEVENV_PERF_REGRESSION_MODE:-warn}" + + if [ "$mode" = "off" ]; then + jq -n --argjson schemaVersion 1 --arg status skipped --arg mode "$mode" '{schemaVersion:$schemaVersion, status:$status, mode:$mode, checks:{}}' >"$ARTIFACT_DIR/perf-comparison.json" + return 0 + fi + + if [ ! -f "$baseline_path" ]; then + jq -n \ + --argjson schemaVersion 1 \ + --arg status baseline_missing \ + --arg mode "$mode" \ + --arg baseline "$baseline_path" \ + '{schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baseline, checks:{}}' \ + >"$ARTIFACT_DIR/perf-comparison.json" + echo "::notice::devenv perf baseline not found at $baseline_path; recorded current measurements only" + return 0 + fi + + jq -n \ + --slurpfile current "$ARTIFACT_DIR/summary.json" \ + --slurpfile baseline "$baseline_path" \ + --argjson schemaVersion 1 \ + --arg mode "$mode" \ + --arg baselinePath "$baseline_path" \ + ' + def budget($name): + if $name == "shell_eval_traced" then + {warnRatio:1.25, failRatio:1.5, warnMs:1500, failMs:3000} + elif $name == "shell_eval_warm" then + {warnRatio:1.5, failRatio:2.0, warnMs:500, failMs:1000} + elif $name == "tasks_list" or $name == "processes_help" then + {warnRatio:2.0, failRatio:3.0, warnMs:250, failMs:1000} + else + {warnRatio:1.5, failRatio:2.0, warnMs:1000, failMs:3000} + end; + def classify($name; $current; $baseline): + budget($name) as $b + | ($current - $baseline) as $delta + | (if $baseline > 0 then ($current / $baseline) else null end) as $ratio + | if $baseline <= 0 then "unknown" + elif ($delta > $b.failMs and $current > ($baseline * $b.failRatio)) then "fail" + elif ($delta > $b.warnMs and $current > ($baseline * $b.warnRatio)) then "warn" + else "pass" + end as $status + | {status:$status, currentMs:$current, baselineMs:$baseline, deltaMs:$delta, ratio:$ratio, budget:$b}; + ($current[0].checks // {}) as $currentChecks + | ($baseline[0].checks // {}) as $baselineChecks + | ( + $currentChecks + | to_entries + | map( + .key as $name + | .value as $current + | ($baselineChecks[$name] // null) as $base + | { + key: $name, + value: + if $base == null then + {status:"missing_baseline", currentMs:$current.durationMs} + elif ($current.status != 0) then + {status:"current_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + elif ($base.status != 0) then + {status:"baseline_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + else + classify($name; $current.durationMs; $base.durationMs) + end + } + ) + | from_entries + ) as $checks + | ( + if any($checks[]; .status == "fail") then "fail" + elif any($checks[]; .status == "warn") then "warn" + elif any($checks[]; .status == "missing_baseline") then "partial" + else "pass" + end + ) as $status + | {schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baselinePath, checks:$checks} + ' >"$ARTIFACT_DIR/perf-comparison.json" + + local status + status="$(jq -r '.status' "$ARTIFACT_DIR/perf-comparison.json")" + case "$status:$mode" in + fail:fail) + echo "::error::devenv perf regression detected" + jq . "$ARTIFACT_DIR/perf-comparison.json" + return 1 + ;; + fail:*|warn:*) + echo "::warning::devenv perf regression threshold exceeded" + jq . "$ARTIFACT_DIR/perf-comparison.json" + ;; + esac +} + +compare_baseline + +if [ -n "${dollar}{GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Devenv perf" + echo "" + echo "| Probe | Status | Duration |" + echo "| --- | ---: | ---: |" + jq -r '.[] | "| \(.name) | \(.status) | \(.durationMs) ms |"' "$ARTIFACT_DIR/timings.json" + echo "" + echo "- Artifact directory: \`$ARTIFACT_DIR\`" + echo "- OTEL service: \`${dollar}{OTEL_SERVICE_NAME:-unknown}\`" + echo "" + echo "#### Regression comparison" + echo "" + if [ -f "$ARTIFACT_DIR/perf-comparison.json" ]; then + jq -r '["- Status: " + .status, "- Mode: " + .mode, "- Baseline: " + (.baseline // "none")] | .[]' "$ARTIFACT_DIR/perf-comparison.json" + fi + } >>"$GITHUB_STEP_SUMMARY" +fi + +cat "$ARTIFACT_DIR/timings.pretty.json" +` +} + +export const devenvPerfBenchmarkStep = ( + opts?: Pick, +) => + ({ + name: 'Benchmark devenv surfaces', + shell: 'bash', + run: renderDevenvPerfScript({ + taskProbes: opts?.taskProbes ?? [], + probes: opts?.probes ?? [], + }), + }) as const + +export const devenvPerfArtifactStep = ( + opts?: Pick, +) => + ({ + name: 'Upload devenv perf artifacts', + if: 'always()', + uses: 'actions/upload-artifact@v4', + with: { + name: + opts?.artifactName ?? + 'devenv-perf-${{ github.job }}-${{ github.run_id }}-attempt-${{ github.run_attempt }}', + path: opts?.artifactDir ?? 'tmp/devenv-perf-ci', + 'if-no-files-found': 'error', + 'retention-days': opts?.retentionDays ?? 30, + }, + }) as const + +export const nixClosureMeasurementStep = (opts: NixClosureMeasurementStepOptions) => { + const artifactDir = opts.artifactDir ?? 'tmp/ci-measurements' + const artifactFile = opts.artifactFile ?? '$ARTIFACT_DIR/measurements.json' + const targetName = opts.targetName ?? opts.installable + const buckets = JSON.stringify(opts.buckets ?? []) + const targetSystemAssignment = + opts.targetSystem === undefined + ? `target_system="${dollar}{DEVENV_SYSTEM:-${dollar}{RUNNER_OS:-unknown}}"` + : `target_system=${shellSingleQuote(opts.targetSystem)}` + + return { + name: `Measure Nix closure: ${targetName}`, + shell: 'bash', + env: { + ARTIFACT_DIR: artifactDir, + RUNNER_CLASS: '${{ runner.os }}-${{ runner.arch }}', + }, + run: String.raw`set -euo pipefail + +mkdir -p "$ARTIFACT_DIR" +installable=${shellSingleQuote(opts.installable)} +target_name=${shellSingleQuote(targetName)} +artifact_file=${shellSingleQuote(artifactFile)} +${targetSystemAssignment} + +out_path="$(nix build --no-link --print-out-paths "$installable")" +path_info="$ARTIFACT_DIR/nix-closure-path-info.json" +paths_file="$ARTIFACT_DIR/nix-closure-paths.json" + +nix path-info --recursive --json "$out_path" >"$path_info" +jq 'to_entries | map({ path: .key, narSize: (.value.narSize // 0) })' "$path_info" >"$paths_file" + +jq -n \ + --slurpfile paths "$paths_file" \ + --argjson schemaVersion 1 \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${dollar}{GITHUB_REPOSITORY:-unknown}" \ + --arg branchKind "${dollar}{GITHUB_EVENT_NAME:-unknown}" \ + --arg ref "${dollar}{GITHUB_REF:-unknown}" \ + --arg headSha "${dollar}{GITHUB_SHA:-unknown}" \ + --arg baseSha "${dollar}{GITHUB_BASE_SHA:-}" \ + --arg runnerName "${dollar}{RUNNER_NAME:-unknown}" \ + --arg runnerOs "${dollar}{RUNNER_OS:-unknown}" \ + --arg runnerArch "${dollar}{RUNNER_ARCH:-unknown}" \ + --arg runnerClass "${dollar}{RUNNER_CLASS:-unknown}" \ + --arg githubRunId "${dollar}{GITHUB_RUN_ID:-unknown}" \ + --arg githubRunAttempt "${dollar}{GITHUB_RUN_ATTEMPT:-unknown}" \ + --arg githubJob "${dollar}{GITHUB_JOB:-unknown}" \ + --arg taskId "${dollar}{CROSSTASK_TASK_ID:-}" \ + --arg taskAttemptId "${dollar}{CROSSTASK_ATTEMPT_ID:-}" \ + --arg traceId "${dollar}{TRACE_ID:-}" \ + --arg targetName "$target_name" \ + --arg targetSystem "$target_system" \ + --arg outPath "$out_path" \ + --argjson buckets ${shellSingleQuote(buckets)} \ + ' + ($paths[0] // []) as $closurePaths + | ($closurePaths | map(.narSize) | add // 0) as $totalNarSize + | ($closurePaths | length) as $pathCount + | ($buckets | map( + . as $bucket + | { + name: "nix.closure.bucket.nar_size", + unit: "bytes", + value: ( + $closurePaths + | map(select(.path | test($bucket.pathRegex)) | .narSize) + | add // 0 + ), + dimensions: { bucket: $bucket.name } + } + )) as $bucketObservations + | { + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + producer: { name: "effect-utils-ci-measurement", version: 1 }, + subject: { + repo: $repository, + branchKind: (if $branchKind == "" then "unknown" else $branchKind end), + ref: $ref, + headSha: $headSha, + baseSha: $baseSha + }, + execution: { + provider: (if ($githubRunId != "" and $githubRunId != "unknown") then "github-actions" else "local" end), + workflow: "CI", + job: $githubJob, + runId: $githubRunId, + runAttempt: $githubRunAttempt, + taskId: $taskId, + attemptId: $taskAttemptId, + traceId: $traceId, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch, class: $runnerClass } + }, + target: { kind: "nix-closure", name: $targetName, system: $targetSystem }, + observations: ([ + { + name: "nix.closure.nar_size", + unit: "bytes", + value: $totalNarSize, + dimensions: { bucket: "total" } + }, + { + name: "nix.closure.path_count", + unit: "count", + value: $pathCount, + dimensions: { bucket: "total" } + } + ] + $bucketObservations), + artifacts: [ + { name: "nix-closure-path-info", path: "nix-closure-path-info.json", contentType: "application/json" }, + { name: "nix-closure-paths", path: "nix-closure-paths.json", contentType: "application/json" } + ], + details: { + outPath: $outPath, + topPaths: ($closurePaths | sort_by(.narSize) | reverse | .[:30]) + } + } + ' >"$artifact_file" + +cat "$artifact_file" +`, + } as const +} + +export const devenvPerfJob = (opts?: DevenvPerfJobOptions) => { + const artifactDir = opts?.artifactDir ?? 'tmp/devenv-perf-ci' + + return { + 'runs-on': opts?.runsOn ?? linuxX64Runner, + defaults: bashShellDefaults, + env: { + ...standardCIEnv, + ARTIFACT_DIR: artifactDir, + OTEL_SERVICE_NAME: 'devenv-perf-ci', + DEVENV_PERF_REGRESSION_MODE: opts?.regressionMode ?? 'warn', + RUNNER_CLASS: (opts?.runsOn ?? linuxX64Runner).join(','), + ...(opts?.env ?? {}), + }, + steps: [ + ...(opts?.setupSteps ?? [ + checkoutStep(), + installNixStep(), + preparePinnedDevenvStep, + validateNixStoreStep, + ]), + devenvPerfBenchmarkStep({ + taskProbes: opts?.taskProbes, + probes: opts?.probes, + }), + devenvPerfArtifactStep({ + artifactDir, + artifactName: opts?.artifactName, + retentionDays: opts?.retentionDays, + }), + ], + } as const +} + const evictOutPathShellLines = [ ' if nix path-info "$outPath" >/dev/null 2>&1; then', ' echo "evicting cached: $(basename "$outPath")"', diff --git a/genie/external.ts b/genie/external.ts index c2643743b..583cc4168 100644 --- a/genie/external.ts +++ b/genie/external.ts @@ -636,6 +636,9 @@ export { cachixStep, cachixBinaryCache, devenvBinaryCache, + devenvPerfArtifactStep, + devenvPerfBenchmarkStep, + devenvPerfJob, pnpmStateSetupStep, restorePnpmStateStep, savePnpmStateStep, @@ -651,6 +654,8 @@ export { syncMegarepoWorkspaceStep, applyMegarepoLockStep, RUNNER_PROFILES, + type DevenvPerfJobOptions, + type DevenvPerfProbe, type NixBinaryCache, type RunnerProfile, } from './ci-workflow.ts' diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 242d86b05..4dc6fb868 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -50,7 +50,11 @@ let "${config.devenv.root}/.devenv/pnpm-home" else "${config.devenv.root}/.devenv/pnpm-home/${workspaceCacheName}"; - defaultPnpmStoreDir = "${config.devenv.root}/.devenv/pnpm-store"; + defaultPnpmStoreDir = + if workspaceRoot == "." then + "${config.devenv.root}/.devenv/pnpm-store" + else + "${config.devenv.root}/.devenv/pnpm-store/${workspaceCacheName}"; installTaskName = if taskSuffix == null then "${taskNamePrefix}:install" @@ -140,8 +144,20 @@ let fi ''; ensureLocalPnpmStoreDirFn = '' - if [ -n "''${npm_config_store_dir:-}" ]; then + if [ ${lib.escapeShellArg workspaceRoot} != "." ] && [ -n "''${npm_config_store_dir:-}" ]; then + case "$npm_config_store_dir" in + */${workspaceCacheName}) ;; + *) export npm_config_store_dir="$npm_config_store_dir/${workspaceCacheName}" ;; + esac + export PNPM_STORE_DIR="$npm_config_store_dir" + elif [ -n "''${npm_config_store_dir:-}" ]; then export PNPM_STORE_DIR="''${PNPM_STORE_DIR:-$npm_config_store_dir}" + elif [ ${lib.escapeShellArg workspaceRoot} != "." ] && [ -n "''${PNPM_STORE_DIR:-}" ]; then + case "$PNPM_STORE_DIR" in + */${workspaceCacheName}) ;; + *) export PNPM_STORE_DIR="$PNPM_STORE_DIR/${workspaceCacheName}" ;; + esac + export npm_config_store_dir="$PNPM_STORE_DIR" elif [ -n "''${PNPM_STORE_DIR:-}" ]; then export npm_config_store_dir="$PNPM_STORE_DIR" else diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index 3f1ed250f..d2350eb13 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -409,7 +409,7 @@ echo "Test 11: status misses after effective store-dir changes" echo "Test 12: exec invoked pnpm install" grep -q "^install " "$tmpdir/pnpm.log" -echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" +echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and store-dir" ( cd "$workspace" export HOME="$tmpdir/home" @@ -422,8 +422,8 @@ echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and sha test -d "$workspace/nested/node_modules" grep -qxF "PWD=$workspace/nested" "$tmpdir/pnpm.log" grep -qxF "PNPM_HOME=$workspace/.devenv/pnpm-home/nested" "$tmpdir/pnpm.log" - grep -qxF "PNPM_STORE_DIR=$workspace/.devenv/pnpm-store" "$tmpdir/pnpm.log" - grep -qxF "npm_config_store_dir=$workspace/.devenv/pnpm-store" "$tmpdir/pnpm.log" + grep -qxF "PNPM_STORE_DIR=$workspace/.devenv/pnpm-store/nested" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$workspace/.devenv/pnpm-store/nested" "$tmpdir/pnpm.log" ) echo "Test 14: nested workspace status hits after nested install" @@ -440,7 +440,20 @@ echo "Test 14: nested workspace status hits after nested install" assert_exit_code 0 "$exit_code" "nested status should hit after nested install" ) -echo "Test 15: install flags and pre-install hooks are applied" +echo "Test 15: nested workspace suffixes an inherited store-dir" +( + cd "$workspace" + export HOME="$tmpdir/home" + unset PNPM_HOME + export PNPM_STORE_DIR="$workspace/.inherited-pnpm-store" + unset npm_config_store_dir + : > "$tmpdir/pnpm.log" + bash "$tmpdir/pnpm-install-nested.exec.sh" + grep -qxF "PNPM_STORE_DIR=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" +) + +echo "Test 16: install flags and pre-install hooks are applied" ( cd "$workspace" export HOME="$tmpdir/home" @@ -454,7 +467,7 @@ echo "Test 15: install flags and pre-install hooks are applied" grep -qxF "install --config.confirmModulesPurge=false --config.store-dir=$workspace/.devenv/pnpm-store --ignore-scripts --config.public-hoist-pattern=*" "$tmpdir/pnpm.log" ) -echo "Test 16: CI install failures preserve and classify the pnpm log" +echo "Test 17: CI install failures preserve and classify the pnpm log" ( cd "$workspace" export HOME="$tmpdir/home" @@ -478,21 +491,21 @@ echo "Test 16: CI install failures preserve and classify the pnpm log" grep -qF "Socket timeout" <<< "$output" ) -echo "Test 17: generated test task runs vitest without pnpm exec" +echo "Test 18: generated test task runs vitest without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/test-demo.exec.sh")" [ "$output" = "vitest-shim:run" ] ) -echo "Test 18: generated storybook task runs storybook without pnpm exec" +echo "Test 19: generated storybook task runs storybook without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/storybook-demo.exec.sh")" [ "$output" = "storybook-shim:build" ] ) -echo "Test 19: clean leaves shared GVS links intact" +echo "Test 20: clean leaves shared GVS links intact" ( cd "$workspace" mkdir -p "$workspace/.devenv/pnpm-store/v11/links/shared-pkg" diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index 3142f277e..126dca2bf 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -10,6 +10,10 @@ const generatedWorkflowSource = readFileSync( new URL(['../../../../../../.github/workflows', 'ci.yml.genie.ts'].join('/'), import.meta.url), 'utf8', ) +const generatedCiWorkflowYamlSource = readFileSync( + new URL(['../../../../../../.github/workflows', 'ci.yml'].join('/'), import.meta.url), + 'utf8', +) const vercelDeploySource = readFileSync( new URL(['../../../../../../genie/deploy-preview', 'vercel.ts'].join('/'), import.meta.url), 'utf8', @@ -262,3 +266,52 @@ describe('ci workflow shared auth helpers', () => { expect(vercelDeploySource).toContain('vercelDeployStep(project, opts.runDevenvTasksBefore)') }) }) + +describe('ci workflow devenv perf helpers', () => { + it('exposes reusable devenv perf CI job helpers', () => { + expect(ciWorkflowSource).toContain('export const devenvPerfJob') + expect(ciWorkflowSource).toContain('export const devenvPerfBenchmarkStep') + expect(ciWorkflowSource).toContain('export const devenvPerfArtifactStep') + expect(ciWorkflowSource).toContain('export type DevenvPerfProbe') + expect(ciWorkflowSource).toContain('export const nixClosureMeasurementStep') + expect(ciWorkflowSource).toContain('export type NixClosureMeasurementBucket') + }) + + it('emits the standard warm shell and task-list probes with native trace artifacts', () => { + expect(generatedCiWorkflowYamlSource).toContain('devenv-perf:') + expect(generatedCiWorkflowYamlSource).toContain('OTEL_SERVICE_NAME: devenv-perf-ci') + expect(generatedCiWorkflowYamlSource).toContain("measure 'shell_eval_traced'") + expect(generatedCiWorkflowYamlSource).toContain('--trace-to') + expect(generatedCiWorkflowYamlSource).toContain('json:file:$trace_file') + expect(generatedCiWorkflowYamlSource).toContain('$ARTIFACT_DIR/traces/shell_eval_traced.json') + expect(generatedCiWorkflowYamlSource).toContain("measure 'shell_eval_warm'") + expect(generatedCiWorkflowYamlSource).toContain("measure 'tasks_list'") + }) + + it('writes a stable summary artifact for regression tracking', () => { + expect(generatedCiWorkflowYamlSource).toContain('schemaVersion: $schemaVersion') + expect(generatedCiWorkflowYamlSource).toContain('checks: ($timings[0] | map') + expect(generatedCiWorkflowYamlSource).toContain('measurements.json') + expect(generatedCiWorkflowYamlSource).toContain('--argjson schemaVersion 1') + expect(generatedCiWorkflowYamlSource).toContain('effect-utils-ci-measurement') + expect(generatedCiWorkflowYamlSource).toContain('devenv." + .name + ".duration') + expect(generatedCiWorkflowYamlSource).toContain( + 'target: { kind: "devenv", name: "dev-shell", system: $targetSystem }', + ) + expect(generatedCiWorkflowYamlSource).toContain('RUNNER_CLASS:') + expect(generatedCiWorkflowYamlSource).toContain('namespace-profile-linux-x86-64') + expect(ciWorkflowSource).toContain('nix.closure.nar_size') + expect(ciWorkflowSource).toContain('nix.closure.path_count') + expect(ciWorkflowSource).toContain('nix.closure.bucket.nar_size') + expect(ciWorkflowSource).toContain('target: { kind: "nix-closure"') + expect(ciWorkflowSource).toContain('nix path-info --recursive --json "$out_path"') + expect(ciWorkflowSource).toContain( + 'topPaths: ($closurePaths | sort_by(.narSize) | reverse | .[:30])', + ) + expect(generatedCiWorkflowYamlSource).not.toContain('dev3') + expect(generatedCiWorkflowYamlSource).toContain('perf-comparison.json') + expect(generatedCiWorkflowYamlSource).toContain('DEVENV_PERF_REGRESSION_MODE') + expect(generatedCiWorkflowYamlSource).toContain('Upload devenv perf artifacts') + expect(generatedCiWorkflowYamlSource).toContain('retention-days: 30') + }) +})