From 3040394e0a85cfceecfc12788fb15324949decdc Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 04:39:49 +0200 Subject: [PATCH 01/10] Add reusable devenv perf workflow helper --- .github/workflows/devenv-perf.yml | 306 ++++++++++++++++++ .github/workflows/devenv-perf.yml.genie.ts | 28 ++ genie/ci-workflow.ts | 259 +++++++++++++++ genie/external.ts | 6 + .../ci-workflow-helpers.unit.test.ts | 30 ++ 5 files changed, 629 insertions(+) create mode 100644 .github/workflows/devenv-perf.yml create mode 100644 .github/workflows/devenv-perf.yml.genie.ts diff --git a/.github/workflows/devenv-perf.yml b/.github/workflows/devenv-perf.yml new file mode 100644 index 000000000..85404a6cb --- /dev/null +++ b/.github/workflows/devenv-perf.yml @@ -0,0 +1,306 @@ +# Generated file - DO NOT EDIT +# Source: devenv-perf.yml.genie.ts + +concurrency: + group: '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}' + cancel-in-progress: true + +name: Devenv Perf + +on: + workflow_dispatch: {} + schedule: + - cron: 17 3 * * * + +jobs: + devenv-perf: + runs-on: [sh-linux-x64, nix] + defaults: + run: + shell: bash + env: + FORCE_SETUP: '1' + CI: 'true' + GITHUB_TOKEN: ${{ github.token }} + ARTIFACT_DIR: tmp/devenv-perf-ci + OTEL_SERVICE_NAME: devenv-perf-ci + steps: + - uses: actions/checkout@v6 + - name: Install Nix + uses: DeterminateSystems/determinate-nix-action@v3 + with: + extra-conf: | + experimental-features = nix-command flakes + accept-flake-config = true + extra-substituters = https://devenv.cachix.org + extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= + access-tokens = github.com=${{ github.token }} + summarize: true + - name: Enable Cachix cache + uses: cachix/cachix-action@v17 + with: + name: overeng-effect-utils + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Use pinned devenv from lock + run: | + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then + echo '::error::devenv.lock missing .nodes.devenv.locked.rev' + exit 1 + fi + echo "DEVENV_REV=$DEVENV_REV" >> "$GITHUB_ENV" + echo "Pinned devenv rev: $DEVENV_REV" + shell: bash + - name: Isolate pnpm state + shell: bash + run: | + echo "PNPM_STORE_DIR=${{ runner.temp }}/pnpm-store/${{ github.job }}" >> "$GITHUB_ENV" + echo "PNPM_HOME=${{ github.workspace }}/.pnpm-home" >> "$GITHUB_ENV" + - id: restore-pnpm-state + name: Restore pnpm state + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.pnpm-home + ${{ runner.temp }}/pnpm-store/${{ github.job }} + key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" + - name: Resolve devenv + run: | + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then + echo '::error::devenv.lock missing .nodes.devenv.locked.rev' + exit 1 + fi + + resolve_devenv() { + nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" + } + + # Temporary: capture diagnostics dir for #272 root-cause analysis. + DIAG_ROOT="${RUNNER_TEMP:-/tmp}/nix-store-diagnostics-${GITHUB_JOB:-job}-${RUNNER_OS:-unknown}-${GITHUB_RUN_ATTEMPT:-0}" + mkdir -p "$DIAG_ROOT" + echo "NIX_STORE_DIAGNOSTICS_DIR=$DIAG_ROOT" >> "$GITHUB_ENV" + + { + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "github_job=${GITHUB_JOB:-unknown}" + echo "github_run_id=${GITHUB_RUN_ID:-unknown}" + echo "nix_user_conf_files=${NIX_USER_CONF_FILES:-}" + nix --version || true + } > "$DIAG_ROOT/environment.txt" 2>&1 + + if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv.log" >&2)); then + echo "::error::resolve_devenv failed. Last 30 lines of log:" + tail -30 "$DIAG_ROOT/resolve-devenv.log" || true + exit 1 + fi + DEVENV_BIN="$DEVENV_OUT/bin/devenv" + + # Fast validity check on the devenv store path (~1-2s vs ~25s for devenv info). + if ! nix-store --check-validity "$DEVENV_OUT" 2>/dev/null; then + echo "::warning::devenv store path invalid, repairing targeted path..." + nix-store --repair-path "$DEVENV_OUT" > "$DIAG_ROOT/nix-store-verify-repair.log" 2>&1 || true + rm -rf "${XDG_CACHE_HOME:-$HOME/.cache}"/nix/eval-cache-* ~/.cache/nix/eval-cache-* + if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv-post-repair.log" >&2)); then + echo "::error::resolve_devenv failed after repair. Last 30 lines of log:" + tail -30 "$DIAG_ROOT/resolve-devenv-post-repair.log" || true + exit 1 + fi + DEVENV_BIN="$DEVENV_OUT/bin/devenv" + fi + + echo "DEVENV_BIN=$DEVENV_BIN" >> "$GITHUB_ENV" + "$DEVENV_BIN" version | tee "$DIAG_ROOT/devenv-version.txt" + shell: bash + - name: Evict cached pnpm deps for oxlint-npm + shell: bash + run: | + targetRef='.#oxlint-npm' + entriesJson=$(mktemp) + if nix eval --json "$targetRef.passthru.depsBuildEntries" >"$entriesJson" 2>/dev/null; then + while IFS=$'\t' read -r attrName drv; do + [ -n "$drv" ] || continue + while IFS= read -r outPath; do + [ -n "$outPath" ] || continue + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "evicting cached: $(basename "$outPath")" + if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then + echo "::error::failed to evict cached pnpm-deps output: $outPath" + exit 1 + fi + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "::error::cached pnpm-deps output still present after eviction: $outPath" + exit 1 + fi + fi + done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) + done < <(jq -r '.[] | [.attrName, (.drvPath // "")] | @tsv' "$entriesJson") + else + topDrv=$(nix path-info --derivation "$targetRef" 2>/dev/null || true) + if [ -n "$topDrv" ]; then + while IFS= read -r drv; do + [ -n "$drv" ] || continue + attrName="" + while IFS= read -r outPath; do + [ -n "$outPath" ] || continue + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "evicting cached: $(basename "$outPath")" + if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then + echo "::error::failed to evict cached pnpm-deps output: $outPath" + exit 1 + fi + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "::error::cached pnpm-deps output still present after eviction: $outPath" + exit 1 + fi + fi + done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) + done < <(nix-store -qR "$topDrv" 2>/dev/null | grep "pnpm-deps-[a-z0-9-]*-v[0-9].*\.drv$" || true) + fi + fi + rm -f "$entriesJson" + - name: Benchmark devenv surfaces + shell: bash + run: | + set -euo pipefail + + mkdir -p "$ARTIFACT_DIR/traces" + + { + printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'repository=%s\n' "${GITHUB_REPOSITORY:-unknown}" + printf 'ref=%s\n' "${GITHUB_REF:-unknown}" + printf 'sha=%s\n' "${GITHUB_SHA:-unknown}" + printf 'runner_name=%s\n' "${RUNNER_NAME:-unknown}" + printf 'runner_os=%s\n' "${RUNNER_OS:-unknown}" + printf 'runner_arch=%s\n' "${RUNNER_ARCH:-unknown}" + printf 'devenv_rev=%s\n' "${DEVENV_REV:-unknown}" + printf 'otel_service_name=%s\n' "${OTEL_SERVICE_NAME:-unknown}" + df -h / /nix 2>/dev/null || df -h / + ps -eo pid,ppid,stat,etime,pcpu,pmem,comm,args 2>/dev/null \ + | grep -E 'devenv direnv-export|nix-daemon|nix build|nix flake|github-runner' \ + | grep -v grep || true + } >"$ARTIFACT_DIR/host-context.txt" + + printf '[' >"$ARTIFACT_DIR/timings.json" + first=1 + + json_append_timing() { + local name="$1" + local status="$2" + local duration_ms="$3" + local stdout="$4" + local stderr="$5" + local trace="$6" + + if [ "$first" -eq 0 ]; then + printf ',' >>"$ARTIFACT_DIR/timings.json" + fi + first=0 + + jq -cn \ + --arg name "$name" \ + --argjson status "$status" \ + --argjson durationMs "$duration_ms" \ + --arg stdout "$stdout" \ + --arg stderr "$stderr" \ + --arg trace "$trace" \ + '{name:$name,status:$status,durationMs:$durationMs,stdout:$stdout,stderr:$stderr,trace:(if $trace == "" then null else $trace end)}' \ + >>"$ARTIFACT_DIR/timings.json" + } + + measure() { + local name="$1" + local trace_file="$2" + shift 2 + local stdout="$ARTIFACT_DIR/$name.stdout" + local stderr="$ARTIFACT_DIR/$name.stderr" + local started ended status duration_ms + + mkdir -p "$(dirname "$trace_file")" + started="$(date +%s%3N)" + set +e + expanded=() + for arg in "$@"; do + case "$arg" in + '$DEVENV_BIN') expanded+=("${DEVENV_BIN:?DEVENV_BIN not set}") ;; + '$ARTIFACT_DIR'*) expanded+=("${ARTIFACT_DIR}${arg#'$ARTIFACT_DIR'}") ;; + '$trace_file') expanded+=("file:$trace_file") ;; + *) expanded+=("$arg") ;; + esac + done + "${expanded[@]}" >"$stdout" 2>"$stderr" + status=$? + set -e + ended="$(date +%s%3N)" + duration_ms=$((ended - started)) + + json_append_timing "$name" "$status" "$duration_ms" "$stdout" "$stderr" "$trace_file" + + if [ "$status" -ne 0 ]; then + echo "::error::$name failed after ${duration_ms}ms; stderr tail follows" + tail -80 "$stderr" || true + return "$status" + fi + } + + measure 'shell_eval_traced' '$ARTIFACT_DIR/traces/shell_eval_traced.json' '$DEVENV_BIN' '--trace-output' '$trace_file' '--trace-format' 'json' 'shell' '--no-reload' '--' 'true' + measure 'shell_eval_warm' '' '$DEVENV_BIN' 'shell' '--no-reload' '--' 'true' + measure 'tasks_list' '' '$DEVENV_BIN' 'tasks' 'list' + measure 'processes_help' '' '$DEVENV_BIN' 'processes' '--help' + measure 'task_otel_test' '' '$DEVENV_BIN' 'tasks' 'run' 'otel:test' '--mode' 'before' '--no-tui' '--show-output' + measure 'task_check_quick' '' '$DEVENV_BIN' 'tasks' 'run' 'check:quick' '--mode' 'before' '--no-tui' '--show-output' + + printf ']\n' >>"$ARTIFACT_DIR/timings.json" + + jq . "$ARTIFACT_DIR/timings.json" >"$ARTIFACT_DIR/timings.pretty.json" + jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --arg schemaVersion "1" \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${GITHUB_REPOSITORY:-unknown}" \ + --arg ref "${GITHUB_REF:-unknown}" \ + --arg sha "${GITHUB_SHA:-unknown}" \ + --arg runnerName "${RUNNER_NAME:-unknown}" \ + --arg runnerOs "${RUNNER_OS:-unknown}" \ + --arg runnerArch "${RUNNER_ARCH:-unknown}" \ + --arg devenvRev "${DEVENV_REV:-unknown}" \ + --arg otelServiceName "${OTEL_SERVICE_NAME:-unknown}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + repository: $repository, + ref: $ref, + sha: $sha, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch }, + devenv: { rev: $devenvRev }, + otel: { serviceName: $otelServiceName }, + checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) + }' >"$ARTIFACT_DIR/summary.json" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Devenv perf" + echo "" + echo "| Probe | Status | Duration |" + echo "| --- | ---: | ---: |" + jq -r '.[] | "| \(.name) | \(.status) | \(.durationMs) ms |"' "$ARTIFACT_DIR/timings.json" + echo "" + echo "- Artifact directory: \`$ARTIFACT_DIR\`" + echo "- OTEL service: \`${OTEL_SERVICE_NAME:-unknown}\`" + } >>"$GITHUB_STEP_SUMMARY" + fi + + cat "$ARTIFACT_DIR/timings.pretty.json" + + - name: Upload devenv perf artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: 'devenv-perf-${{ github.job }}-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: tmp/devenv-perf-ci + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/devenv-perf.yml.genie.ts b/.github/workflows/devenv-perf.yml.genie.ts new file mode 100644 index 000000000..55fb22b2b --- /dev/null +++ b/.github/workflows/devenv-perf.yml.genie.ts @@ -0,0 +1,28 @@ +import { + cachixStep, + checkoutStep, + devenvPerfWorkflow, + evictCachedPnpmDepsStep, + installNixStep, + pnpmStateSetupStep, + preparePinnedDevenvStep, + restorePnpmStateStep, + validateNixStoreStep, +} from '../../genie/ci-workflow.ts' + +export default devenvPerfWorkflow({ + setupSteps: [ + checkoutStep(), + installNixStep(), + cachixStep({ name: 'overeng-effect-utils', authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' }), + preparePinnedDevenvStep, + pnpmStateSetupStep, + restorePnpmStateStep(), + validateNixStoreStep, + evictCachedPnpmDepsStep({ + flakeRef: '.#oxlint-npm', + name: 'Evict cached pnpm deps for oxlint-npm', + }), + ], + taskProbes: ['otel:test', 'check:quick'], +}) diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index bbaf243de..94aecdb3d 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -348,6 +348,265 @@ export const runDevenvTasksBefore = (...args: [string, ...string[]]) => label: `devenv tasks run ${args.join(' ')} --mode before`, }) +export type DevenvPerfProbe = { + readonly name: string + readonly command: readonly [string, ...string[]] + readonly traceOutput?: string +} + +type DevenvPerfSetupStep = GitHubWorkflowArgs['jobs'][string]['steps'][number] + +export type DevenvPerfJobOptions = { + readonly runsOn?: readonly string[] + readonly artifactDir?: string + readonly artifactName?: string + readonly setupSteps?: readonly DevenvPerfSetupStep[] + readonly env?: Record + readonly taskProbes?: readonly string[] + readonly probes?: readonly DevenvPerfProbe[] + readonly retentionDays?: number +} + +const devenvPerfProbeLine = (probe: DevenvPerfProbe) => { + const args = probe.command.map(shellSingleQuote).join(' ') + const trace = probe.traceOutput ?? '' + return `measure ${shellSingleQuote(probe.name)} ${shellSingleQuote(trace)} ${args}` +} + +const defaultDevenvPerfTaskProbe = (task: string): DevenvPerfProbe => ({ + name: `task_${task.replaceAll(':', '_')}`, + command: [ + '$DEVENV_BIN', + 'tasks', + 'run', + task, + '--mode', + 'before', + '--no-tui', + '--show-output', + ], +}) + +const renderDevenvPerfScript = (opts: Required>) => { + const probes: readonly DevenvPerfProbe[] = [ + { + name: 'shell_eval_traced', + command: [ + '$DEVENV_BIN', + '--trace-output', + '$trace_file', + '--trace-format', + 'json', + 'shell', + '--no-reload', + '--', + 'true', + ], + traceOutput: '$ARTIFACT_DIR/traces/shell_eval_traced.json', + }, + { name: 'shell_eval_warm', command: ['$DEVENV_BIN', 'shell', '--no-reload', '--', 'true'] }, + { name: 'tasks_list', command: ['$DEVENV_BIN', 'tasks', 'list'] }, + { name: 'processes_help', command: ['$DEVENV_BIN', 'processes', '--help'] }, + ...opts.taskProbes.map(defaultDevenvPerfTaskProbe), + ...opts.probes, + ] + + return String.raw`set -euo pipefail + +mkdir -p "$ARTIFACT_DIR/traces" + +{ + printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'repository=%s\n' "${dollar}{GITHUB_REPOSITORY:-unknown}" + printf 'ref=%s\n' "${dollar}{GITHUB_REF:-unknown}" + printf 'sha=%s\n' "${dollar}{GITHUB_SHA:-unknown}" + printf 'runner_name=%s\n' "${dollar}{RUNNER_NAME:-unknown}" + printf 'runner_os=%s\n' "${dollar}{RUNNER_OS:-unknown}" + printf 'runner_arch=%s\n' "${dollar}{RUNNER_ARCH:-unknown}" + printf 'devenv_rev=%s\n' "${dollar}{DEVENV_REV:-unknown}" + printf 'otel_service_name=%s\n' "${dollar}{OTEL_SERVICE_NAME:-unknown}" + df -h / /nix 2>/dev/null || df -h / + ps -eo pid,ppid,stat,etime,pcpu,pmem,comm,args 2>/dev/null \ + | grep -E 'devenv direnv-export|nix-daemon|nix build|nix flake|github-runner' \ + | grep -v grep || true +} >"$ARTIFACT_DIR/host-context.txt" + +printf '[' >"$ARTIFACT_DIR/timings.json" +first=1 + +json_append_timing() { + local name="$1" + local status="$2" + local duration_ms="$3" + local stdout="$4" + local stderr="$5" + local trace="$6" + + if [ "$first" -eq 0 ]; then + printf ',' >>"$ARTIFACT_DIR/timings.json" + fi + first=0 + + jq -cn \ + --arg name "$name" \ + --argjson status "$status" \ + --argjson durationMs "$duration_ms" \ + --arg stdout "$stdout" \ + --arg stderr "$stderr" \ + --arg trace "$trace" \ + '{name:$name,status:$status,durationMs:$durationMs,stdout:$stdout,stderr:$stderr,trace:(if $trace == "" then null else $trace end)}' \ + >>"$ARTIFACT_DIR/timings.json" +} + +measure() { + local name="$1" + local trace_file="$2" + shift 2 + local stdout="$ARTIFACT_DIR/$name.stdout" + local stderr="$ARTIFACT_DIR/$name.stderr" + local started ended status duration_ms + + mkdir -p "$(dirname "$trace_file")" + started="$(date +%s%3N)" + set +e + expanded=() + for arg in "$@"; do + case "$arg" in + '$DEVENV_BIN') expanded+=("${dollar}{DEVENV_BIN:?DEVENV_BIN not set}") ;; + '$ARTIFACT_DIR'*) expanded+=("${dollar}{ARTIFACT_DIR}${dollar}{arg#'$ARTIFACT_DIR'}") ;; + '$trace_file') expanded+=("file:$trace_file") ;; + *) expanded+=("$arg") ;; + esac + done + "${dollar}{expanded[@]}" >"$stdout" 2>"$stderr" + status=$? + set -e + ended="$(date +%s%3N)" + duration_ms=$((ended - started)) + + json_append_timing "$name" "$status" "$duration_ms" "$stdout" "$stderr" "$trace_file" + + if [ "$status" -ne 0 ]; then + echo "::error::$name failed after ${dollar}{duration_ms}ms; stderr tail follows" + tail -80 "$stderr" || true + return "$status" + fi +} + +${probes.map(devenvPerfProbeLine).join('\n')} + +printf ']\n' >>"$ARTIFACT_DIR/timings.json" + +jq . "$ARTIFACT_DIR/timings.json" >"$ARTIFACT_DIR/timings.pretty.json" +jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --arg schemaVersion "1" \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${dollar}{GITHUB_REPOSITORY:-unknown}" \ + --arg ref "${dollar}{GITHUB_REF:-unknown}" \ + --arg sha "${dollar}{GITHUB_SHA:-unknown}" \ + --arg runnerName "${dollar}{RUNNER_NAME:-unknown}" \ + --arg runnerOs "${dollar}{RUNNER_OS:-unknown}" \ + --arg runnerArch "${dollar}{RUNNER_ARCH:-unknown}" \ + --arg devenvRev "${dollar}{DEVENV_REV:-unknown}" \ + --arg otelServiceName "${dollar}{OTEL_SERVICE_NAME:-unknown}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + repository: $repository, + ref: $ref, + sha: $sha, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch }, + devenv: { rev: $devenvRev }, + otel: { serviceName: $otelServiceName }, + checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) + }' >"$ARTIFACT_DIR/summary.json" + +if [ -n "${dollar}{GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Devenv perf" + echo "" + echo "| Probe | Status | Duration |" + echo "| --- | ---: | ---: |" + jq -r '.[] | "| \(.name) | \(.status) | \(.durationMs) ms |"' "$ARTIFACT_DIR/timings.json" + echo "" + echo "- Artifact directory: \`$ARTIFACT_DIR\`" + echo "- OTEL service: \`${dollar}{OTEL_SERVICE_NAME:-unknown}\`" + } >>"$GITHUB_STEP_SUMMARY" +fi + +cat "$ARTIFACT_DIR/timings.pretty.json" +` +} + +export const devenvPerfBenchmarkStep = (opts?: Pick) => + ({ + name: 'Benchmark devenv surfaces', + shell: 'bash', + run: renderDevenvPerfScript({ + taskProbes: opts?.taskProbes ?? [], + probes: opts?.probes ?? [], + }), + }) as const + +export const devenvPerfArtifactStep = ( + opts?: Pick, +) => + ({ + name: 'Upload devenv perf artifacts', + if: 'always()', + uses: 'actions/upload-artifact@v4', + with: { + name: + opts?.artifactName ?? + 'devenv-perf-${{ github.job }}-${{ github.run_id }}-attempt-${{ github.run_attempt }}', + path: opts?.artifactDir ?? 'tmp/devenv-perf-ci', + 'if-no-files-found': 'error', + 'retention-days': opts?.retentionDays ?? 30, + }, + }) as const + +export const devenvPerfJob = (opts?: DevenvPerfJobOptions) => { + const artifactDir = opts?.artifactDir ?? 'tmp/devenv-perf-ci' + + return { + 'runs-on': opts?.runsOn ?? linuxX64Runner, + defaults: bashShellDefaults, + env: { + ...standardCIEnv, + ARTIFACT_DIR: artifactDir, + OTEL_SERVICE_NAME: 'devenv-perf-ci', + ...(opts?.env ?? {}), + }, + steps: [ + ...(opts?.setupSteps ?? [checkoutStep(), installNixStep(), preparePinnedDevenvStep, validateNixStoreStep]), + devenvPerfBenchmarkStep({ + taskProbes: opts?.taskProbes, + probes: opts?.probes, + }), + devenvPerfArtifactStep({ + artifactDir, + artifactName: opts?.artifactName, + retentionDays: opts?.retentionDays, + }), + ], + } as const +} + +export const devenvPerfWorkflow = ( + opts?: Omit & { readonly name?: string }, +) => + ciWorkflow({ + name: opts?.name ?? 'Devenv Perf', + on: { + workflow_dispatch: {}, + schedule: [{ cron: '17 3 * * *' }], + }, + jobs: { + 'devenv-perf': devenvPerfJob(opts), + }, + }) + const evictOutPathShellLines = [ ' if nix path-info "$outPath" >/dev/null 2>&1; then', ' echo "evicting cached: $(basename "$outPath")"', diff --git a/genie/external.ts b/genie/external.ts index c2643743b..82f4832c8 100644 --- a/genie/external.ts +++ b/genie/external.ts @@ -636,6 +636,10 @@ export { cachixStep, cachixBinaryCache, devenvBinaryCache, + devenvPerfArtifactStep, + devenvPerfBenchmarkStep, + devenvPerfJob, + devenvPerfWorkflow, pnpmStateSetupStep, restorePnpmStateStep, savePnpmStateStep, @@ -651,6 +655,8 @@ export { syncMegarepoWorkspaceStep, applyMegarepoLockStep, RUNNER_PROFILES, + type DevenvPerfJobOptions, + type DevenvPerfProbe, type NixBinaryCache, type RunnerProfile, } from './ci-workflow.ts' diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index 3142f277e..28c859864 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -10,6 +10,10 @@ const generatedWorkflowSource = readFileSync( new URL(['../../../../../../.github/workflows', 'ci.yml.genie.ts'].join('/'), import.meta.url), 'utf8', ) +const generatedDevenvPerfWorkflowSource = readFileSync( + new URL(['../../../../../../.github/workflows', 'devenv-perf.yml'].join('/'), import.meta.url), + 'utf8', +) const vercelDeploySource = readFileSync( new URL(['../../../../../../genie/deploy-preview', 'vercel.ts'].join('/'), import.meta.url), 'utf8', @@ -262,3 +266,29 @@ describe('ci workflow shared auth helpers', () => { expect(vercelDeploySource).toContain('vercelDeployStep(project, opts.runDevenvTasksBefore)') }) }) + +describe('ci workflow devenv perf helpers', () => { + it('exposes a reusable devenv perf workflow helper', () => { + expect(ciWorkflowSource).toContain('export const devenvPerfJob') + expect(ciWorkflowSource).toContain('export const devenvPerfWorkflow') + expect(ciWorkflowSource).toContain('export type DevenvPerfProbe') + }) + + it('emits the standard warm shell and task-list probes with native trace artifacts', () => { + expect(generatedDevenvPerfWorkflowSource).toContain('OTEL_SERVICE_NAME: devenv-perf-ci') + expect(generatedDevenvPerfWorkflowSource).toContain("measure 'shell_eval_traced'") + expect(generatedDevenvPerfWorkflowSource).toContain('--trace-output') + expect(generatedDevenvPerfWorkflowSource).toContain( + '$ARTIFACT_DIR/traces/shell_eval_traced.json', + ) + expect(generatedDevenvPerfWorkflowSource).toContain("measure 'shell_eval_warm'") + expect(generatedDevenvPerfWorkflowSource).toContain("measure 'tasks_list'") + }) + + it('writes a stable summary artifact for regression tracking', () => { + expect(generatedDevenvPerfWorkflowSource).toContain('schemaVersion: $schemaVersion') + expect(generatedDevenvPerfWorkflowSource).toContain('checks: ($timings[0] | map') + expect(generatedDevenvPerfWorkflowSource).toContain('Upload devenv perf artifacts') + expect(generatedDevenvPerfWorkflowSource).toContain('retention-days: 30') + }) +}) From acf19a0ed5d02792ba865e936a310251e1407a8f Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 05:01:19 +0200 Subject: [PATCH 02/10] Isolate nested pnpm stores --- nix/devenv-modules/tasks/shared/pnpm.nix | 6 +++++- .../tasks/shared/tests/pnpm-task-smoke.test.sh | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 242d86b05..9817b9c93 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -50,7 +50,11 @@ let "${config.devenv.root}/.devenv/pnpm-home" else "${config.devenv.root}/.devenv/pnpm-home/${workspaceCacheName}"; - defaultPnpmStoreDir = "${config.devenv.root}/.devenv/pnpm-store"; + defaultPnpmStoreDir = + if workspaceRoot == "." then + "${config.devenv.root}/.devenv/pnpm-store" + else + "${config.devenv.root}/.devenv/pnpm-store/${workspaceCacheName}"; installTaskName = if taskSuffix == null then "${taskNamePrefix}:install" diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index 3f1ed250f..5d395b4c3 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -409,7 +409,7 @@ echo "Test 11: status misses after effective store-dir changes" echo "Test 12: exec invoked pnpm install" grep -q "^install " "$tmpdir/pnpm.log" -echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and shared store-dir" +echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and store-dir" ( cd "$workspace" export HOME="$tmpdir/home" @@ -422,8 +422,8 @@ echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and sha test -d "$workspace/nested/node_modules" grep -qxF "PWD=$workspace/nested" "$tmpdir/pnpm.log" grep -qxF "PNPM_HOME=$workspace/.devenv/pnpm-home/nested" "$tmpdir/pnpm.log" - grep -qxF "PNPM_STORE_DIR=$workspace/.devenv/pnpm-store" "$tmpdir/pnpm.log" - grep -qxF "npm_config_store_dir=$workspace/.devenv/pnpm-store" "$tmpdir/pnpm.log" + grep -qxF "PNPM_STORE_DIR=$workspace/.devenv/pnpm-store/nested" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$workspace/.devenv/pnpm-store/nested" "$tmpdir/pnpm.log" ) echo "Test 14: nested workspace status hits after nested install" From b48552c315e1297ba9c214413a2a1a1bf8209fe0 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 05:09:26 +0200 Subject: [PATCH 03/10] Isolate inherited pnpm store paths --- nix/devenv-modules/tasks/shared/pnpm.nix | 14 ++++++++++- .../shared/tests/pnpm-task-smoke.test.sh | 23 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 9817b9c93..4dc6fb868 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -144,8 +144,20 @@ let fi ''; ensureLocalPnpmStoreDirFn = '' - if [ -n "''${npm_config_store_dir:-}" ]; then + if [ ${lib.escapeShellArg workspaceRoot} != "." ] && [ -n "''${npm_config_store_dir:-}" ]; then + case "$npm_config_store_dir" in + */${workspaceCacheName}) ;; + *) export npm_config_store_dir="$npm_config_store_dir/${workspaceCacheName}" ;; + esac + export PNPM_STORE_DIR="$npm_config_store_dir" + elif [ -n "''${npm_config_store_dir:-}" ]; then export PNPM_STORE_DIR="''${PNPM_STORE_DIR:-$npm_config_store_dir}" + elif [ ${lib.escapeShellArg workspaceRoot} != "." ] && [ -n "''${PNPM_STORE_DIR:-}" ]; then + case "$PNPM_STORE_DIR" in + */${workspaceCacheName}) ;; + *) export PNPM_STORE_DIR="$PNPM_STORE_DIR/${workspaceCacheName}" ;; + esac + export npm_config_store_dir="$PNPM_STORE_DIR" elif [ -n "''${PNPM_STORE_DIR:-}" ]; then export npm_config_store_dir="$PNPM_STORE_DIR" else diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index 5d395b4c3..d2350eb13 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -440,7 +440,20 @@ echo "Test 14: nested workspace status hits after nested install" assert_exit_code 0 "$exit_code" "nested status should hit after nested install" ) -echo "Test 15: install flags and pre-install hooks are applied" +echo "Test 15: nested workspace suffixes an inherited store-dir" +( + cd "$workspace" + export HOME="$tmpdir/home" + unset PNPM_HOME + export PNPM_STORE_DIR="$workspace/.inherited-pnpm-store" + unset npm_config_store_dir + : > "$tmpdir/pnpm.log" + bash "$tmpdir/pnpm-install-nested.exec.sh" + grep -qxF "PNPM_STORE_DIR=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" +) + +echo "Test 16: install flags and pre-install hooks are applied" ( cd "$workspace" export HOME="$tmpdir/home" @@ -454,7 +467,7 @@ echo "Test 15: install flags and pre-install hooks are applied" grep -qxF "install --config.confirmModulesPurge=false --config.store-dir=$workspace/.devenv/pnpm-store --ignore-scripts --config.public-hoist-pattern=*" "$tmpdir/pnpm.log" ) -echo "Test 16: CI install failures preserve and classify the pnpm log" +echo "Test 17: CI install failures preserve and classify the pnpm log" ( cd "$workspace" export HOME="$tmpdir/home" @@ -478,21 +491,21 @@ echo "Test 16: CI install failures preserve and classify the pnpm log" grep -qF "Socket timeout" <<< "$output" ) -echo "Test 17: generated test task runs vitest without pnpm exec" +echo "Test 18: generated test task runs vitest without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/test-demo.exec.sh")" [ "$output" = "vitest-shim:run" ] ) -echo "Test 18: generated storybook task runs storybook without pnpm exec" +echo "Test 19: generated storybook task runs storybook without pnpm exec" ( cd "$workspace/packages/demo" output="$(bash "$tmpdir/storybook-demo.exec.sh")" [ "$output" = "storybook-shim:build" ] ) -echo "Test 19: clean leaves shared GVS links intact" +echo "Test 20: clean leaves shared GVS links intact" ( cd "$workspace" mkdir -p "$workspace/.devenv/pnpm-store/v11/links/shared-pkg" From 7774891cc561429fa5704cbcccf9a879f331ec65 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 07:29:27 +0200 Subject: [PATCH 04/10] Fold devenv perf into CI --- .github/workflows/ci.yml | 306 ++++++++++++++++++ .github/workflows/ci.yml.genie.ts | 9 + .github/workflows/devenv-perf.yml | 306 ------------------ .github/workflows/devenv-perf.yml.genie.ts | 28 -- genie/ci-workflow.ts | 14 - genie/external.ts | 1 - .../ci-workflow-helpers.unit.test.ts | 32 +- 7 files changed, 331 insertions(+), 365 deletions(-) delete mode 100644 .github/workflows/devenv-perf.yml delete mode 100644 .github/workflows/devenv-perf.yml.genie.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8350205f3..65810cf2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2078,6 +2078,312 @@ jobs: run: | echo "If this looks like Namespace runner Nix store corruption (e.g. \"... is not valid\", \"config.cachix\", \"cachix.package\"), add the run link + full nix-store output to:" echo " https://github.com/overengineeringstudio/effect-utils/issues/201" + devenv-perf: + runs-on: + [namespace-profile-linux-x86-64, 'namespace-features:github.run-id=${{ github.run_id }}'] + defaults: + run: + shell: bash + env: + FORCE_SETUP: '1' + CI: 'true' + GITHUB_TOKEN: ${{ github.token }} + ARTIFACT_DIR: tmp/devenv-perf-ci + OTEL_SERVICE_NAME: devenv-perf-ci + steps: + - uses: actions/checkout@v6 + - name: Install Nix + uses: DeterminateSystems/determinate-nix-action@v3 + with: + extra-conf: | + experimental-features = nix-command flakes + accept-flake-config = true + extra-substituters = https://devenv.cachix.org + extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= + access-tokens = github.com=${{ github.token }} + summarize: true + - name: Enable Cachix cache + uses: cachix/cachix-action@v17 + with: + name: overeng-effect-utils + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Use pinned devenv from lock + run: | + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then + echo '::error::devenv.lock missing .nodes.devenv.locked.rev' + exit 1 + fi + echo "DEVENV_REV=$DEVENV_REV" >> "$GITHUB_ENV" + echo "Pinned devenv rev: $DEVENV_REV" + shell: bash + - name: Isolate pnpm state + shell: bash + run: | + echo "PNPM_STORE_DIR=${{ runner.temp }}/pnpm-store/${{ github.job }}" >> "$GITHUB_ENV" + echo "PNPM_HOME=${{ github.workspace }}/.pnpm-home" >> "$GITHUB_ENV" + - id: restore-pnpm-state + name: Restore pnpm state + uses: actions/cache/restore@v4 + with: + path: | + ${{ github.workspace }}/.pnpm-home + ${{ runner.temp }}/pnpm-store/${{ github.job }} + key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" + - name: Resolve devenv + run: | + DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) + if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then + echo '::error::devenv.lock missing .nodes.devenv.locked.rev' + exit 1 + fi + + resolve_devenv() { + nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" + } + + # Temporary: capture diagnostics dir for #272 root-cause analysis. + DIAG_ROOT="${RUNNER_TEMP:-/tmp}/nix-store-diagnostics-${GITHUB_JOB:-job}-${RUNNER_OS:-unknown}-${GITHUB_RUN_ATTEMPT:-0}" + mkdir -p "$DIAG_ROOT" + echo "NIX_STORE_DIAGNOSTICS_DIR=$DIAG_ROOT" >> "$GITHUB_ENV" + + { + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "runner_name=${RUNNER_NAME:-unknown}" + echo "runner_os=${RUNNER_OS:-unknown}" + echo "runner_arch=${RUNNER_ARCH:-unknown}" + echo "github_job=${GITHUB_JOB:-unknown}" + echo "github_run_id=${GITHUB_RUN_ID:-unknown}" + echo "nix_user_conf_files=${NIX_USER_CONF_FILES:-}" + nix --version || true + } > "$DIAG_ROOT/environment.txt" 2>&1 + + if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv.log" >&2)); then + echo "::error::resolve_devenv failed. Last 30 lines of log:" + tail -30 "$DIAG_ROOT/resolve-devenv.log" || true + exit 1 + fi + DEVENV_BIN="$DEVENV_OUT/bin/devenv" + + # Fast validity check on the devenv store path (~1-2s vs ~25s for devenv info). + if ! nix-store --check-validity "$DEVENV_OUT" 2>/dev/null; then + echo "::warning::devenv store path invalid, repairing targeted path..." + nix-store --repair-path "$DEVENV_OUT" > "$DIAG_ROOT/nix-store-verify-repair.log" 2>&1 || true + rm -rf "${XDG_CACHE_HOME:-$HOME/.cache}"/nix/eval-cache-* ~/.cache/nix/eval-cache-* + if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv-post-repair.log" >&2)); then + echo "::error::resolve_devenv failed after repair. Last 30 lines of log:" + tail -30 "$DIAG_ROOT/resolve-devenv-post-repair.log" || true + exit 1 + fi + DEVENV_BIN="$DEVENV_OUT/bin/devenv" + fi + + echo "DEVENV_BIN=$DEVENV_BIN" >> "$GITHUB_ENV" + "$DEVENV_BIN" version | tee "$DIAG_ROOT/devenv-version.txt" + shell: bash + - name: Evict cached pnpm deps for oxlint-npm + shell: bash + run: | + targetRef='.#oxlint-npm' + entriesJson=$(mktemp) + if nix eval --json "$targetRef.passthru.depsBuildEntries" >"$entriesJson" 2>/dev/null; then + while IFS=$'\t' read -r attrName drv; do + [ -n "$drv" ] || continue + while IFS= read -r outPath; do + [ -n "$outPath" ] || continue + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "evicting cached: $(basename "$outPath")" + if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then + echo "::error::failed to evict cached pnpm-deps output: $outPath" + exit 1 + fi + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "::error::cached pnpm-deps output still present after eviction: $outPath" + exit 1 + fi + fi + done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) + done < <(jq -r '.[] | [.attrName, (.drvPath // "")] | @tsv' "$entriesJson") + else + topDrv=$(nix path-info --derivation "$targetRef" 2>/dev/null || true) + if [ -n "$topDrv" ]; then + while IFS= read -r drv; do + [ -n "$drv" ] || continue + attrName="" + while IFS= read -r outPath; do + [ -n "$outPath" ] || continue + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "evicting cached: $(basename "$outPath")" + if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then + echo "::error::failed to evict cached pnpm-deps output: $outPath" + exit 1 + fi + if nix path-info "$outPath" >/dev/null 2>&1; then + echo "::error::cached pnpm-deps output still present after eviction: $outPath" + exit 1 + fi + fi + done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) + done < <(nix-store -qR "$topDrv" 2>/dev/null | grep "pnpm-deps-[a-z0-9-]*-v[0-9].*\.drv$" || true) + fi + fi + rm -f "$entriesJson" + - name: Force diagnostics failure (debug) + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.debug_force_nix_diagnostics_failure == true || inputs.debug_force_nix_diagnostics_failure == 'true') }} + shell: bash + run: | + diag_dir="${NIX_STORE_DIAGNOSTICS_DIR:-${RUNNER_TEMP:-/tmp}/nix-store-diagnostics-missing}" + mkdir -p "$diag_dir" + cat > "$diag_dir/synthetic-signature.log" <<'EOF' + Failed to convert config.cachix to JSON + ... while evaluating the option `cachix.package` + error: path '/nix/store/synthetic-invalid-path' is not valid + EOF + echo "::warning::Intentional failure for diagnostics validation (#272)" + exit 1 + - name: Benchmark devenv surfaces + shell: bash + run: | + set -euo pipefail + + mkdir -p "$ARTIFACT_DIR/traces" + + { + printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'repository=%s\n' "${GITHUB_REPOSITORY:-unknown}" + printf 'ref=%s\n' "${GITHUB_REF:-unknown}" + printf 'sha=%s\n' "${GITHUB_SHA:-unknown}" + printf 'runner_name=%s\n' "${RUNNER_NAME:-unknown}" + printf 'runner_os=%s\n' "${RUNNER_OS:-unknown}" + printf 'runner_arch=%s\n' "${RUNNER_ARCH:-unknown}" + printf 'devenv_rev=%s\n' "${DEVENV_REV:-unknown}" + printf 'otel_service_name=%s\n' "${OTEL_SERVICE_NAME:-unknown}" + df -h / /nix 2>/dev/null || df -h / + ps -eo pid,ppid,stat,etime,pcpu,pmem,comm,args 2>/dev/null \ + | grep -E 'devenv direnv-export|nix-daemon|nix build|nix flake|github-runner' \ + | grep -v grep || true + } >"$ARTIFACT_DIR/host-context.txt" + + printf '[' >"$ARTIFACT_DIR/timings.json" + first=1 + + json_append_timing() { + local name="$1" + local status="$2" + local duration_ms="$3" + local stdout="$4" + local stderr="$5" + local trace="$6" + + if [ "$first" -eq 0 ]; then + printf ',' >>"$ARTIFACT_DIR/timings.json" + fi + first=0 + + jq -cn \ + --arg name "$name" \ + --argjson status "$status" \ + --argjson durationMs "$duration_ms" \ + --arg stdout "$stdout" \ + --arg stderr "$stderr" \ + --arg trace "$trace" \ + '{name:$name,status:$status,durationMs:$durationMs,stdout:$stdout,stderr:$stderr,trace:(if $trace == "" then null else $trace end)}' \ + >>"$ARTIFACT_DIR/timings.json" + } + + measure() { + local name="$1" + local trace_file="$2" + shift 2 + local stdout="$ARTIFACT_DIR/$name.stdout" + local stderr="$ARTIFACT_DIR/$name.stderr" + local started ended status duration_ms + + mkdir -p "$(dirname "$trace_file")" + started="$(date +%s%3N)" + set +e + expanded=() + for arg in "$@"; do + case "$arg" in + '$DEVENV_BIN') expanded+=("${DEVENV_BIN:?DEVENV_BIN not set}") ;; + '$ARTIFACT_DIR'*) expanded+=("${ARTIFACT_DIR}${arg#'$ARTIFACT_DIR'}") ;; + '$trace_file') expanded+=("file:$trace_file") ;; + *) expanded+=("$arg") ;; + esac + done + "${expanded[@]}" >"$stdout" 2>"$stderr" + status=$? + set -e + ended="$(date +%s%3N)" + duration_ms=$((ended - started)) + + json_append_timing "$name" "$status" "$duration_ms" "$stdout" "$stderr" "$trace_file" + + if [ "$status" -ne 0 ]; then + echo "::error::$name failed after ${duration_ms}ms; stderr tail follows" + tail -80 "$stderr" || true + return "$status" + fi + } + + measure 'shell_eval_traced' '$ARTIFACT_DIR/traces/shell_eval_traced.json' '$DEVENV_BIN' '--trace-output' '$trace_file' '--trace-format' 'json' 'shell' '--no-reload' '--' 'true' + measure 'shell_eval_warm' '' '$DEVENV_BIN' 'shell' '--no-reload' '--' 'true' + measure 'tasks_list' '' '$DEVENV_BIN' 'tasks' 'list' + measure 'processes_help' '' '$DEVENV_BIN' 'processes' '--help' + measure 'task_pnpm_install' '' '$DEVENV_BIN' 'tasks' 'run' 'pnpm:install' '--mode' 'before' '--no-tui' '--show-output' + measure 'task_genie_run' '' '$DEVENV_BIN' 'tasks' 'run' 'genie:run' '--mode' 'before' '--no-tui' '--show-output' + measure 'task_check_quick' '' '$DEVENV_BIN' 'tasks' 'run' 'check:quick' '--mode' 'before' '--no-tui' '--show-output' + + printf ']\n' >>"$ARTIFACT_DIR/timings.json" + + jq . "$ARTIFACT_DIR/timings.json" >"$ARTIFACT_DIR/timings.pretty.json" + jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --arg schemaVersion "1" \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${GITHUB_REPOSITORY:-unknown}" \ + --arg ref "${GITHUB_REF:-unknown}" \ + --arg sha "${GITHUB_SHA:-unknown}" \ + --arg runnerName "${RUNNER_NAME:-unknown}" \ + --arg runnerOs "${RUNNER_OS:-unknown}" \ + --arg runnerArch "${RUNNER_ARCH:-unknown}" \ + --arg devenvRev "${DEVENV_REV:-unknown}" \ + --arg otelServiceName "${OTEL_SERVICE_NAME:-unknown}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + repository: $repository, + ref: $ref, + sha: $sha, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch }, + devenv: { rev: $devenvRev }, + otel: { serviceName: $otelServiceName }, + checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) + }' >"$ARTIFACT_DIR/summary.json" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Devenv perf" + echo "" + echo "| Probe | Status | Duration |" + echo "| --- | ---: | ---: |" + jq -r '.[] | "| \(.name) | \(.status) | \(.durationMs) ms |"' "$ARTIFACT_DIR/timings.json" + echo "" + echo "- Artifact directory: \`$ARTIFACT_DIR\`" + echo "- OTEL service: \`${OTEL_SERVICE_NAME:-unknown}\`" + } >>"$GITHUB_STEP_SUMMARY" + fi + + cat "$ARTIFACT_DIR/timings.pretty.json" + + - name: Upload devenv perf artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: 'devenv-perf-${{ github.job }}-${{ github.run_id }}-attempt-${{ github.run_attempt }}' + path: tmp/devenv-perf-ci + if-no-files-found: error + retention-days: 30 test-integration-notion: runs-on: [namespace-profile-linux-x86-64, 'namespace-features:github.run-id=${{ github.run_id }}'] diff --git a/.github/workflows/ci.yml.genie.ts b/.github/workflows/ci.yml.genie.ts index c4faf73f1..fca8ec180 100644 --- a/.github/workflows/ci.yml.genie.ts +++ b/.github/workflows/ci.yml.genie.ts @@ -14,6 +14,7 @@ import { savePnpmStateStep, standardCIEnv, ciWorkflow, + devenvPerfJob, namespaceRunner, validateColdPnpmDepsStep, nixDiagnosticsArtifactStep, @@ -251,6 +252,14 @@ const NETLIFY_SITE = 'overeng-utils' // Non-required jobs (separate from CIJobName — not required status checks) const extraJobs: Record = { + 'devenv-perf': devenvPerfJob({ + runsOn: namespaceRunner({ + profile: 'namespace-profile-linux-x86-64', + runId: '${{ github.run_id }}', + }), + setupSteps: baseSteps, + taskProbes: ['pnpm:install', 'genie:run', 'check:quick'], + }), /** Integration tests for Notion API (requires NOTION_TOKEN secret) */ 'test-integration-notion': { 'runs-on': namespaceRunner({ diff --git a/.github/workflows/devenv-perf.yml b/.github/workflows/devenv-perf.yml deleted file mode 100644 index 85404a6cb..000000000 --- a/.github/workflows/devenv-perf.yml +++ /dev/null @@ -1,306 +0,0 @@ -# Generated file - DO NOT EDIT -# Source: devenv-perf.yml.genie.ts - -concurrency: - group: '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}' - cancel-in-progress: true - -name: Devenv Perf - -on: - workflow_dispatch: {} - schedule: - - cron: 17 3 * * * - -jobs: - devenv-perf: - runs-on: [sh-linux-x64, nix] - defaults: - run: - shell: bash - env: - FORCE_SETUP: '1' - CI: 'true' - GITHUB_TOKEN: ${{ github.token }} - ARTIFACT_DIR: tmp/devenv-perf-ci - OTEL_SERVICE_NAME: devenv-perf-ci - steps: - - uses: actions/checkout@v6 - - name: Install Nix - uses: DeterminateSystems/determinate-nix-action@v3 - with: - extra-conf: | - experimental-features = nix-command flakes - accept-flake-config = true - extra-substituters = https://devenv.cachix.org - extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= - access-tokens = github.com=${{ github.token }} - summarize: true - - name: Enable Cachix cache - uses: cachix/cachix-action@v17 - with: - name: overeng-effect-utils - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - - name: Use pinned devenv from lock - run: | - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) - if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then - echo '::error::devenv.lock missing .nodes.devenv.locked.rev' - exit 1 - fi - echo "DEVENV_REV=$DEVENV_REV" >> "$GITHUB_ENV" - echo "Pinned devenv rev: $DEVENV_REV" - shell: bash - - name: Isolate pnpm state - shell: bash - run: | - echo "PNPM_STORE_DIR=${{ runner.temp }}/pnpm-store/${{ github.job }}" >> "$GITHUB_ENV" - echo "PNPM_HOME=${{ github.workspace }}/.pnpm-home" >> "$GITHUB_ENV" - - id: restore-pnpm-state - name: Restore pnpm state - uses: actions/cache/restore@v4 - with: - path: | - ${{ github.workspace }}/.pnpm-home - ${{ runner.temp }}/pnpm-store/${{ github.job }} - key: "pnpm-state-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pnpm-lock.yaml') }}" - - name: Resolve devenv - run: | - DEVENV_REV=$(jq -r .nodes.devenv.locked.rev devenv.lock) - if [ -z "$DEVENV_REV" ] || [ "$DEVENV_REV" = "null" ]; then - echo '::error::devenv.lock missing .nodes.devenv.locked.rev' - exit 1 - fi - - resolve_devenv() { - nix build --no-link --print-out-paths "github:cachix/devenv/$DEVENV_REV#devenv" - } - - # Temporary: capture diagnostics dir for #272 root-cause analysis. - DIAG_ROOT="${RUNNER_TEMP:-/tmp}/nix-store-diagnostics-${GITHUB_JOB:-job}-${RUNNER_OS:-unknown}-${GITHUB_RUN_ATTEMPT:-0}" - mkdir -p "$DIAG_ROOT" - echo "NIX_STORE_DIAGNOSTICS_DIR=$DIAG_ROOT" >> "$GITHUB_ENV" - - { - echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "runner_name=${RUNNER_NAME:-unknown}" - echo "runner_os=${RUNNER_OS:-unknown}" - echo "runner_arch=${RUNNER_ARCH:-unknown}" - echo "github_job=${GITHUB_JOB:-unknown}" - echo "github_run_id=${GITHUB_RUN_ID:-unknown}" - echo "nix_user_conf_files=${NIX_USER_CONF_FILES:-}" - nix --version || true - } > "$DIAG_ROOT/environment.txt" 2>&1 - - if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv.log" >&2)); then - echo "::error::resolve_devenv failed. Last 30 lines of log:" - tail -30 "$DIAG_ROOT/resolve-devenv.log" || true - exit 1 - fi - DEVENV_BIN="$DEVENV_OUT/bin/devenv" - - # Fast validity check on the devenv store path (~1-2s vs ~25s for devenv info). - if ! nix-store --check-validity "$DEVENV_OUT" 2>/dev/null; then - echo "::warning::devenv store path invalid, repairing targeted path..." - nix-store --repair-path "$DEVENV_OUT" > "$DIAG_ROOT/nix-store-verify-repair.log" 2>&1 || true - rm -rf "${XDG_CACHE_HOME:-$HOME/.cache}"/nix/eval-cache-* ~/.cache/nix/eval-cache-* - if ! DEVENV_OUT=$(resolve_devenv 2> >(tee "$DIAG_ROOT/resolve-devenv-post-repair.log" >&2)); then - echo "::error::resolve_devenv failed after repair. Last 30 lines of log:" - tail -30 "$DIAG_ROOT/resolve-devenv-post-repair.log" || true - exit 1 - fi - DEVENV_BIN="$DEVENV_OUT/bin/devenv" - fi - - echo "DEVENV_BIN=$DEVENV_BIN" >> "$GITHUB_ENV" - "$DEVENV_BIN" version | tee "$DIAG_ROOT/devenv-version.txt" - shell: bash - - name: Evict cached pnpm deps for oxlint-npm - shell: bash - run: | - targetRef='.#oxlint-npm' - entriesJson=$(mktemp) - if nix eval --json "$targetRef.passthru.depsBuildEntries" >"$entriesJson" 2>/dev/null; then - while IFS=$'\t' read -r attrName drv; do - [ -n "$drv" ] || continue - while IFS= read -r outPath; do - [ -n "$outPath" ] || continue - if nix path-info "$outPath" >/dev/null 2>&1; then - echo "evicting cached: $(basename "$outPath")" - if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then - echo "::error::failed to evict cached pnpm-deps output: $outPath" - exit 1 - fi - if nix path-info "$outPath" >/dev/null 2>&1; then - echo "::error::cached pnpm-deps output still present after eviction: $outPath" - exit 1 - fi - fi - done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) - done < <(jq -r '.[] | [.attrName, (.drvPath // "")] | @tsv' "$entriesJson") - else - topDrv=$(nix path-info --derivation "$targetRef" 2>/dev/null || true) - if [ -n "$topDrv" ]; then - while IFS= read -r drv; do - [ -n "$drv" ] || continue - attrName="" - while IFS= read -r outPath; do - [ -n "$outPath" ] || continue - if nix path-info "$outPath" >/dev/null 2>&1; then - echo "evicting cached: $(basename "$outPath")" - if ! nix store delete --ignore-liveness "$outPath" >/dev/null 2>&1; then - echo "::error::failed to evict cached pnpm-deps output: $outPath" - exit 1 - fi - if nix path-info "$outPath" >/dev/null 2>&1; then - echo "::error::cached pnpm-deps output still present after eviction: $outPath" - exit 1 - fi - fi - done < <(nix-store -q --outputs "$drv" 2>/dev/null || true) - done < <(nix-store -qR "$topDrv" 2>/dev/null | grep "pnpm-deps-[a-z0-9-]*-v[0-9].*\.drv$" || true) - fi - fi - rm -f "$entriesJson" - - name: Benchmark devenv surfaces - shell: bash - run: | - set -euo pipefail - - mkdir -p "$ARTIFACT_DIR/traces" - - { - printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - printf 'repository=%s\n' "${GITHUB_REPOSITORY:-unknown}" - printf 'ref=%s\n' "${GITHUB_REF:-unknown}" - printf 'sha=%s\n' "${GITHUB_SHA:-unknown}" - printf 'runner_name=%s\n' "${RUNNER_NAME:-unknown}" - printf 'runner_os=%s\n' "${RUNNER_OS:-unknown}" - printf 'runner_arch=%s\n' "${RUNNER_ARCH:-unknown}" - printf 'devenv_rev=%s\n' "${DEVENV_REV:-unknown}" - printf 'otel_service_name=%s\n' "${OTEL_SERVICE_NAME:-unknown}" - df -h / /nix 2>/dev/null || df -h / - ps -eo pid,ppid,stat,etime,pcpu,pmem,comm,args 2>/dev/null \ - | grep -E 'devenv direnv-export|nix-daemon|nix build|nix flake|github-runner' \ - | grep -v grep || true - } >"$ARTIFACT_DIR/host-context.txt" - - printf '[' >"$ARTIFACT_DIR/timings.json" - first=1 - - json_append_timing() { - local name="$1" - local status="$2" - local duration_ms="$3" - local stdout="$4" - local stderr="$5" - local trace="$6" - - if [ "$first" -eq 0 ]; then - printf ',' >>"$ARTIFACT_DIR/timings.json" - fi - first=0 - - jq -cn \ - --arg name "$name" \ - --argjson status "$status" \ - --argjson durationMs "$duration_ms" \ - --arg stdout "$stdout" \ - --arg stderr "$stderr" \ - --arg trace "$trace" \ - '{name:$name,status:$status,durationMs:$durationMs,stdout:$stdout,stderr:$stderr,trace:(if $trace == "" then null else $trace end)}' \ - >>"$ARTIFACT_DIR/timings.json" - } - - measure() { - local name="$1" - local trace_file="$2" - shift 2 - local stdout="$ARTIFACT_DIR/$name.stdout" - local stderr="$ARTIFACT_DIR/$name.stderr" - local started ended status duration_ms - - mkdir -p "$(dirname "$trace_file")" - started="$(date +%s%3N)" - set +e - expanded=() - for arg in "$@"; do - case "$arg" in - '$DEVENV_BIN') expanded+=("${DEVENV_BIN:?DEVENV_BIN not set}") ;; - '$ARTIFACT_DIR'*) expanded+=("${ARTIFACT_DIR}${arg#'$ARTIFACT_DIR'}") ;; - '$trace_file') expanded+=("file:$trace_file") ;; - *) expanded+=("$arg") ;; - esac - done - "${expanded[@]}" >"$stdout" 2>"$stderr" - status=$? - set -e - ended="$(date +%s%3N)" - duration_ms=$((ended - started)) - - json_append_timing "$name" "$status" "$duration_ms" "$stdout" "$stderr" "$trace_file" - - if [ "$status" -ne 0 ]; then - echo "::error::$name failed after ${duration_ms}ms; stderr tail follows" - tail -80 "$stderr" || true - return "$status" - fi - } - - measure 'shell_eval_traced' '$ARTIFACT_DIR/traces/shell_eval_traced.json' '$DEVENV_BIN' '--trace-output' '$trace_file' '--trace-format' 'json' 'shell' '--no-reload' '--' 'true' - measure 'shell_eval_warm' '' '$DEVENV_BIN' 'shell' '--no-reload' '--' 'true' - measure 'tasks_list' '' '$DEVENV_BIN' 'tasks' 'list' - measure 'processes_help' '' '$DEVENV_BIN' 'processes' '--help' - measure 'task_otel_test' '' '$DEVENV_BIN' 'tasks' 'run' 'otel:test' '--mode' 'before' '--no-tui' '--show-output' - measure 'task_check_quick' '' '$DEVENV_BIN' 'tasks' 'run' 'check:quick' '--mode' 'before' '--no-tui' '--show-output' - - printf ']\n' >>"$ARTIFACT_DIR/timings.json" - - jq . "$ARTIFACT_DIR/timings.json" >"$ARTIFACT_DIR/timings.pretty.json" - jq -n \ - --slurpfile timings "$ARTIFACT_DIR/timings.json" \ - --arg schemaVersion "1" \ - --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --arg repository "${GITHUB_REPOSITORY:-unknown}" \ - --arg ref "${GITHUB_REF:-unknown}" \ - --arg sha "${GITHUB_SHA:-unknown}" \ - --arg runnerName "${RUNNER_NAME:-unknown}" \ - --arg runnerOs "${RUNNER_OS:-unknown}" \ - --arg runnerArch "${RUNNER_ARCH:-unknown}" \ - --arg devenvRev "${DEVENV_REV:-unknown}" \ - --arg otelServiceName "${OTEL_SERVICE_NAME:-unknown}" \ - '{ - schemaVersion: $schemaVersion, - generatedAt: $generatedAt, - repository: $repository, - ref: $ref, - sha: $sha, - runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch }, - devenv: { rev: $devenvRev }, - otel: { serviceName: $otelServiceName }, - checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) - }' >"$ARTIFACT_DIR/summary.json" - - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - echo "### Devenv perf" - echo "" - echo "| Probe | Status | Duration |" - echo "| --- | ---: | ---: |" - jq -r '.[] | "| \(.name) | \(.status) | \(.durationMs) ms |"' "$ARTIFACT_DIR/timings.json" - echo "" - echo "- Artifact directory: \`$ARTIFACT_DIR\`" - echo "- OTEL service: \`${OTEL_SERVICE_NAME:-unknown}\`" - } >>"$GITHUB_STEP_SUMMARY" - fi - - cat "$ARTIFACT_DIR/timings.pretty.json" - - - name: Upload devenv perf artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: 'devenv-perf-${{ github.job }}-${{ github.run_id }}-attempt-${{ github.run_attempt }}' - path: tmp/devenv-perf-ci - if-no-files-found: error - retention-days: 30 diff --git a/.github/workflows/devenv-perf.yml.genie.ts b/.github/workflows/devenv-perf.yml.genie.ts deleted file mode 100644 index 55fb22b2b..000000000 --- a/.github/workflows/devenv-perf.yml.genie.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { - cachixStep, - checkoutStep, - devenvPerfWorkflow, - evictCachedPnpmDepsStep, - installNixStep, - pnpmStateSetupStep, - preparePinnedDevenvStep, - restorePnpmStateStep, - validateNixStoreStep, -} from '../../genie/ci-workflow.ts' - -export default devenvPerfWorkflow({ - setupSteps: [ - checkoutStep(), - installNixStep(), - cachixStep({ name: 'overeng-effect-utils', authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' }), - preparePinnedDevenvStep, - pnpmStateSetupStep, - restorePnpmStateStep(), - validateNixStoreStep, - evictCachedPnpmDepsStep({ - flakeRef: '.#oxlint-npm', - name: 'Evict cached pnpm deps for oxlint-npm', - }), - ], - taskProbes: ['otel:test', 'check:quick'], -}) diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 94aecdb3d..9b6a8a185 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -593,20 +593,6 @@ export const devenvPerfJob = (opts?: DevenvPerfJobOptions) => { } as const } -export const devenvPerfWorkflow = ( - opts?: Omit & { readonly name?: string }, -) => - ciWorkflow({ - name: opts?.name ?? 'Devenv Perf', - on: { - workflow_dispatch: {}, - schedule: [{ cron: '17 3 * * *' }], - }, - jobs: { - 'devenv-perf': devenvPerfJob(opts), - }, - }) - const evictOutPathShellLines = [ ' if nix path-info "$outPath" >/dev/null 2>&1; then', ' echo "evicting cached: $(basename "$outPath")"', diff --git a/genie/external.ts b/genie/external.ts index 82f4832c8..583cc4168 100644 --- a/genie/external.ts +++ b/genie/external.ts @@ -639,7 +639,6 @@ export { devenvPerfArtifactStep, devenvPerfBenchmarkStep, devenvPerfJob, - devenvPerfWorkflow, pnpmStateSetupStep, restorePnpmStateStep, savePnpmStateStep, diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index 28c859864..eb1af80c5 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -10,8 +10,8 @@ const generatedWorkflowSource = readFileSync( new URL(['../../../../../../.github/workflows', 'ci.yml.genie.ts'].join('/'), import.meta.url), 'utf8', ) -const generatedDevenvPerfWorkflowSource = readFileSync( - new URL(['../../../../../../.github/workflows', 'devenv-perf.yml'].join('/'), import.meta.url), +const generatedCiWorkflowYamlSource = readFileSync( + new URL(['../../../../../../.github/workflows', 'ci.yml'].join('/'), import.meta.url), 'utf8', ) const vercelDeploySource = readFileSync( @@ -268,27 +268,27 @@ describe('ci workflow shared auth helpers', () => { }) describe('ci workflow devenv perf helpers', () => { - it('exposes a reusable devenv perf workflow helper', () => { + it('exposes reusable devenv perf CI job helpers', () => { expect(ciWorkflowSource).toContain('export const devenvPerfJob') - expect(ciWorkflowSource).toContain('export const devenvPerfWorkflow') + expect(ciWorkflowSource).toContain('export const devenvPerfBenchmarkStep') + expect(ciWorkflowSource).toContain('export const devenvPerfArtifactStep') expect(ciWorkflowSource).toContain('export type DevenvPerfProbe') }) it('emits the standard warm shell and task-list probes with native trace artifacts', () => { - expect(generatedDevenvPerfWorkflowSource).toContain('OTEL_SERVICE_NAME: devenv-perf-ci') - expect(generatedDevenvPerfWorkflowSource).toContain("measure 'shell_eval_traced'") - expect(generatedDevenvPerfWorkflowSource).toContain('--trace-output') - expect(generatedDevenvPerfWorkflowSource).toContain( - '$ARTIFACT_DIR/traces/shell_eval_traced.json', - ) - expect(generatedDevenvPerfWorkflowSource).toContain("measure 'shell_eval_warm'") - expect(generatedDevenvPerfWorkflowSource).toContain("measure 'tasks_list'") + expect(generatedCiWorkflowYamlSource).toContain('devenv-perf:') + expect(generatedCiWorkflowYamlSource).toContain('OTEL_SERVICE_NAME: devenv-perf-ci') + expect(generatedCiWorkflowYamlSource).toContain("measure 'shell_eval_traced'") + expect(generatedCiWorkflowYamlSource).toContain('--trace-output') + expect(generatedCiWorkflowYamlSource).toContain('$ARTIFACT_DIR/traces/shell_eval_traced.json') + expect(generatedCiWorkflowYamlSource).toContain("measure 'shell_eval_warm'") + expect(generatedCiWorkflowYamlSource).toContain("measure 'tasks_list'") }) it('writes a stable summary artifact for regression tracking', () => { - expect(generatedDevenvPerfWorkflowSource).toContain('schemaVersion: $schemaVersion') - expect(generatedDevenvPerfWorkflowSource).toContain('checks: ($timings[0] | map') - expect(generatedDevenvPerfWorkflowSource).toContain('Upload devenv perf artifacts') - expect(generatedDevenvPerfWorkflowSource).toContain('retention-days: 30') + expect(generatedCiWorkflowYamlSource).toContain('schemaVersion: $schemaVersion') + expect(generatedCiWorkflowYamlSource).toContain('checks: ($timings[0] | map') + expect(generatedCiWorkflowYamlSource).toContain('Upload devenv perf artifacts') + expect(generatedCiWorkflowYamlSource).toContain('retention-days: 30') }) }) From 4ec752fcc42705670bc32eff0765855c015a97c8 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 12:51:29 +0200 Subject: [PATCH 05/10] Add semantic devenv perf measurements --- .github/workflows/ci.yml | 182 ++++++++++++++ genie/ci-workflow.ts | 223 +++++++++++++++++- .../ci-workflow-helpers.unit.test.ts | 13 + 3 files changed, 405 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65810cf2e..fd379bee9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2090,6 +2090,8 @@ jobs: GITHUB_TOKEN: ${{ github.token }} ARTIFACT_DIR: tmp/devenv-perf-ci OTEL_SERVICE_NAME: devenv-perf-ci + DEVENV_PERF_REGRESSION_MODE: warn + RUNNER_CLASS: 'namespace-profile-linux-x86-64,namespace-features:github.run-id=${{ github.run_id }}' steps: - uses: actions/checkout@v6 - name: Install Nix @@ -2361,6 +2363,180 @@ jobs: checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) }' >"$ARTIFACT_DIR/summary.json" + jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --argjson schemaVersion 1 \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${GITHUB_REPOSITORY:-unknown}" \ + --arg branchKind "${GITHUB_EVENT_NAME:-unknown}" \ + --arg ref "${GITHUB_REF:-unknown}" \ + --arg headSha "${GITHUB_SHA:-unknown}" \ + --arg baseSha "${GITHUB_BASE_SHA:-}" \ + --arg runnerName "${RUNNER_NAME:-unknown}" \ + --arg runnerOs "${RUNNER_OS:-unknown}" \ + --arg runnerArch "${RUNNER_ARCH:-unknown}" \ + --arg runnerClass "${RUNNER_CLASS:-unknown}" \ + --arg githubRunId "${GITHUB_RUN_ID:-unknown}" \ + --arg githubRunAttempt "${GITHUB_RUN_ATTEMPT:-unknown}" \ + --arg githubJob "${GITHUB_JOB:-unknown}" \ + --arg taskId "${CROSSTASK_TASK_ID:-}" \ + --arg taskAttemptId "${CROSSTASK_ATTEMPT_ID:-}" \ + --arg traceId "${TRACE_ID:-}" \ + --arg devenvRev "${DEVENV_REV:-unknown}" \ + --arg otelServiceName "${OTEL_SERVICE_NAME:-unknown}" \ + --arg targetSystem "${DEVENV_SYSTEM:-${RUNNER_OS:-unknown}}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + producer: { name: "effect-utils-ci-measurement", version: 1 }, + subject: { + repo: $repository, + branchKind: (if $branchKind == "" then "unknown" else $branchKind end), + ref: $ref, + headSha: $headSha, + baseSha: $baseSha + }, + execution: { + provider: (if ($githubRunId != "" and $githubRunId != "unknown") then "github-actions" else "local" end), + workflow: "CI", + job: $githubJob, + runId: $githubRunId, + runAttempt: $githubRunAttempt, + taskId: $taskId, + attemptId: $taskAttemptId, + traceId: $traceId, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch, class: $runnerClass } + }, + target: { kind: "devenv", name: "dev-shell", system: $targetSystem }, + observations: ( + $timings[0] + | map({ + name: "devenv." + .name + ".duration", + unit: "seconds", + value: (.durationMs / 1000), + dimensions: { + probe: .name, + status: .status, + devenvRev: $devenvRev, + otelServiceName: $otelServiceName + } + }) + ), + artifacts: [ + { name: "host-context", path: "host-context.txt", contentType: "text/plain" }, + { name: "timings", path: "timings.json", contentType: "application/json" }, + { name: "summary", path: "summary.json", contentType: "application/json" }, + { name: "shell-eval-trace", path: "traces/shell_eval_traced.json", contentType: "application/json" } + ], + details: { + stdoutStderrByProbe: ( + $timings[0] + | map({ key: .name, value: { stdout: .stdout, stderr: .stderr, trace: .trace } }) + | from_entries + ) + } + }' >"$ARTIFACT_DIR/measurements.json" + + compare_baseline() { + local baseline_path="${DEVENV_PERF_BASELINE_SUMMARY:-$ARTIFACT_DIR/baseline/summary.json}" + local mode="${DEVENV_PERF_REGRESSION_MODE:-warn}" + + if [ "$mode" = "off" ]; then + jq -n --argjson schemaVersion 1 --arg status skipped --arg mode "$mode" '{schemaVersion:$schemaVersion, status:$status, mode:$mode, checks:{}}' >"$ARTIFACT_DIR/perf-comparison.json" + return 0 + fi + + if [ ! -f "$baseline_path" ]; then + jq -n \ + --argjson schemaVersion 1 \ + --arg status baseline_missing \ + --arg mode "$mode" \ + --arg baseline "$baseline_path" \ + '{schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baseline, checks:{}}' \ + >"$ARTIFACT_DIR/perf-comparison.json" + echo "::notice::devenv perf baseline not found at $baseline_path; recorded current measurements only" + return 0 + fi + + jq -n \ + --slurpfile current "$ARTIFACT_DIR/summary.json" \ + --slurpfile baseline "$baseline_path" \ + --argjson schemaVersion 1 \ + --arg mode "$mode" \ + --arg baselinePath "$baseline_path" \ + ' + def budget($name): + if $name == "shell_eval_traced" then + {warnRatio:1.25, failRatio:1.5, warnMs:1500, failMs:3000} + elif $name == "shell_eval_warm" then + {warnRatio:1.5, failRatio:2.0, warnMs:500, failMs:1000} + elif $name == "tasks_list" or $name == "processes_help" then + {warnRatio:2.0, failRatio:3.0, warnMs:250, failMs:1000} + else + {warnRatio:1.5, failRatio:2.0, warnMs:1000, failMs:3000} + end; + def classify($name; $current; $baseline): + budget($name) as $b + | ($current - $baseline) as $delta + | (if $baseline > 0 then ($current / $baseline) else null end) as $ratio + | if $baseline <= 0 then "unknown" + elif ($delta > $b.failMs and $current > ($baseline * $b.failRatio)) then "fail" + elif ($delta > $b.warnMs and $current > ($baseline * $b.warnRatio)) then "warn" + else "pass" + end as $status + | {status:$status, currentMs:$current, baselineMs:$baseline, deltaMs:$delta, ratio:$ratio, budget:$b}; + ($current[0].checks // {}) as $currentChecks + | ($baseline[0].checks // {}) as $baselineChecks + | ( + $currentChecks + | to_entries + | map( + .key as $name + | .value as $current + | ($baselineChecks[$name] // null) as $base + | { + key: $name, + value: + if $base == null then + {status:"missing_baseline", currentMs:$current.durationMs} + elif ($current.status != 0) then + {status:"current_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + elif ($base.status != 0) then + {status:"baseline_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + else + classify($name; $current.durationMs; $base.durationMs) + end + } + ) + | from_entries + ) as $checks + | ( + if any($checks[]; .status == "fail") then "fail" + elif any($checks[]; .status == "warn") then "warn" + elif any($checks[]; .status == "missing_baseline") then "partial" + else "pass" + end + ) as $status + | {schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baselinePath, checks:$checks} + ' >"$ARTIFACT_DIR/perf-comparison.json" + + local status + status="$(jq -r '.status' "$ARTIFACT_DIR/perf-comparison.json")" + case "$status:$mode" in + fail:fail) + echo "::error::devenv perf regression detected" + jq . "$ARTIFACT_DIR/perf-comparison.json" + return 1 + ;; + fail:*|warn:*) + echo "::warning::devenv perf regression threshold exceeded" + jq . "$ARTIFACT_DIR/perf-comparison.json" + ;; + esac + } + + compare_baseline + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then { echo "### Devenv perf" @@ -2371,6 +2547,12 @@ jobs: echo "" echo "- Artifact directory: \`$ARTIFACT_DIR\`" echo "- OTEL service: \`${OTEL_SERVICE_NAME:-unknown}\`" + echo "" + echo "#### Regression comparison" + echo "" + if [ -f "$ARTIFACT_DIR/perf-comparison.json" ]; then + jq -r '["- Status: " + .status, "- Mode: " + .mode, "- Baseline: " + (.baseline // "none")] | .[]' "$ARTIFACT_DIR/perf-comparison.json" + fi } >>"$GITHUB_STEP_SUMMARY" fi diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 9b6a8a185..43fcf02e6 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -354,6 +354,20 @@ export type DevenvPerfProbe = { readonly traceOutput?: string } +export type CiMeasurementObservation = { + readonly name: string + readonly unit: string + readonly value: number + readonly dimensions?: Record +} + +export const ciMeasurementMetrics = { + devenvProbeDuration: 'devenv..duration', + nixClosureNarSize: 'nix.closure.nar_size', + nixClosurePathCount: 'nix.closure.path_count', + nixClosureBucketNarSize: 'nix.closure.bucket.nar_size', +} as const + type DevenvPerfSetupStep = GitHubWorkflowArgs['jobs'][string]['steps'][number] export type DevenvPerfJobOptions = { @@ -365,6 +379,7 @@ export type DevenvPerfJobOptions = { readonly taskProbes?: readonly string[] readonly probes?: readonly DevenvPerfProbe[] readonly retentionDays?: number + readonly regressionMode?: 'off' | 'warn' | 'fail' } const devenvPerfProbeLine = (probe: DevenvPerfProbe) => { @@ -375,19 +390,12 @@ const devenvPerfProbeLine = (probe: DevenvPerfProbe) => { const defaultDevenvPerfTaskProbe = (task: string): DevenvPerfProbe => ({ name: `task_${task.replaceAll(':', '_')}`, - command: [ - '$DEVENV_BIN', - 'tasks', - 'run', - task, - '--mode', - 'before', - '--no-tui', - '--show-output', - ], + command: ['$DEVENV_BIN', 'tasks', 'run', task, '--mode', 'before', '--no-tui', '--show-output'], }) -const renderDevenvPerfScript = (opts: Required>) => { +const renderDevenvPerfScript = ( + opts: Required>, +) => { const probes: readonly DevenvPerfProbe[] = [ { name: 'shell_eval_traced', @@ -522,6 +530,180 @@ jq -n \ checks: ($timings[0] | map({ key: .name, value: . }) | from_entries) }' >"$ARTIFACT_DIR/summary.json" +jq -n \ + --slurpfile timings "$ARTIFACT_DIR/timings.json" \ + --argjson schemaVersion 1 \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${dollar}{GITHUB_REPOSITORY:-unknown}" \ + --arg branchKind "${dollar}{GITHUB_EVENT_NAME:-unknown}" \ + --arg ref "${dollar}{GITHUB_REF:-unknown}" \ + --arg headSha "${dollar}{GITHUB_SHA:-unknown}" \ + --arg baseSha "${dollar}{GITHUB_BASE_SHA:-}" \ + --arg runnerName "${dollar}{RUNNER_NAME:-unknown}" \ + --arg runnerOs "${dollar}{RUNNER_OS:-unknown}" \ + --arg runnerArch "${dollar}{RUNNER_ARCH:-unknown}" \ + --arg runnerClass "${dollar}{RUNNER_CLASS:-unknown}" \ + --arg githubRunId "${dollar}{GITHUB_RUN_ID:-unknown}" \ + --arg githubRunAttempt "${dollar}{GITHUB_RUN_ATTEMPT:-unknown}" \ + --arg githubJob "${dollar}{GITHUB_JOB:-unknown}" \ + --arg taskId "${dollar}{CROSSTASK_TASK_ID:-}" \ + --arg taskAttemptId "${dollar}{CROSSTASK_ATTEMPT_ID:-}" \ + --arg traceId "${dollar}{TRACE_ID:-}" \ + --arg devenvRev "${dollar}{DEVENV_REV:-unknown}" \ + --arg otelServiceName "${dollar}{OTEL_SERVICE_NAME:-unknown}" \ + --arg targetSystem "${dollar}{DEVENV_SYSTEM:-${dollar}{RUNNER_OS:-unknown}}" \ + '{ + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + producer: { name: "effect-utils-ci-measurement", version: 1 }, + subject: { + repo: $repository, + branchKind: (if $branchKind == "" then "unknown" else $branchKind end), + ref: $ref, + headSha: $headSha, + baseSha: $baseSha + }, + execution: { + provider: (if ($githubRunId != "" and $githubRunId != "unknown") then "github-actions" else "local" end), + workflow: "CI", + job: $githubJob, + runId: $githubRunId, + runAttempt: $githubRunAttempt, + taskId: $taskId, + attemptId: $taskAttemptId, + traceId: $traceId, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch, class: $runnerClass } + }, + target: { kind: "devenv", name: "dev-shell", system: $targetSystem }, + observations: ( + $timings[0] + | map({ + name: "devenv." + .name + ".duration", + unit: "seconds", + value: (.durationMs / 1000), + dimensions: { + probe: .name, + status: .status, + devenvRev: $devenvRev, + otelServiceName: $otelServiceName + } + }) + ), + artifacts: [ + { name: "host-context", path: "host-context.txt", contentType: "text/plain" }, + { name: "timings", path: "timings.json", contentType: "application/json" }, + { name: "summary", path: "summary.json", contentType: "application/json" }, + { name: "shell-eval-trace", path: "traces/shell_eval_traced.json", contentType: "application/json" } + ], + details: { + stdoutStderrByProbe: ( + $timings[0] + | map({ key: .name, value: { stdout: .stdout, stderr: .stderr, trace: .trace } }) + | from_entries + ) + } + }' >"$ARTIFACT_DIR/measurements.json" + +compare_baseline() { + local baseline_path="${dollar}{DEVENV_PERF_BASELINE_SUMMARY:-$ARTIFACT_DIR/baseline/summary.json}" + local mode="${dollar}{DEVENV_PERF_REGRESSION_MODE:-warn}" + + if [ "$mode" = "off" ]; then + jq -n --argjson schemaVersion 1 --arg status skipped --arg mode "$mode" '{schemaVersion:$schemaVersion, status:$status, mode:$mode, checks:{}}' >"$ARTIFACT_DIR/perf-comparison.json" + return 0 + fi + + if [ ! -f "$baseline_path" ]; then + jq -n \ + --argjson schemaVersion 1 \ + --arg status baseline_missing \ + --arg mode "$mode" \ + --arg baseline "$baseline_path" \ + '{schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baseline, checks:{}}' \ + >"$ARTIFACT_DIR/perf-comparison.json" + echo "::notice::devenv perf baseline not found at $baseline_path; recorded current measurements only" + return 0 + fi + + jq -n \ + --slurpfile current "$ARTIFACT_DIR/summary.json" \ + --slurpfile baseline "$baseline_path" \ + --argjson schemaVersion 1 \ + --arg mode "$mode" \ + --arg baselinePath "$baseline_path" \ + ' + def budget($name): + if $name == "shell_eval_traced" then + {warnRatio:1.25, failRatio:1.5, warnMs:1500, failMs:3000} + elif $name == "shell_eval_warm" then + {warnRatio:1.5, failRatio:2.0, warnMs:500, failMs:1000} + elif $name == "tasks_list" or $name == "processes_help" then + {warnRatio:2.0, failRatio:3.0, warnMs:250, failMs:1000} + else + {warnRatio:1.5, failRatio:2.0, warnMs:1000, failMs:3000} + end; + def classify($name; $current; $baseline): + budget($name) as $b + | ($current - $baseline) as $delta + | (if $baseline > 0 then ($current / $baseline) else null end) as $ratio + | if $baseline <= 0 then "unknown" + elif ($delta > $b.failMs and $current > ($baseline * $b.failRatio)) then "fail" + elif ($delta > $b.warnMs and $current > ($baseline * $b.warnRatio)) then "warn" + else "pass" + end as $status + | {status:$status, currentMs:$current, baselineMs:$baseline, deltaMs:$delta, ratio:$ratio, budget:$b}; + ($current[0].checks // {}) as $currentChecks + | ($baseline[0].checks // {}) as $baselineChecks + | ( + $currentChecks + | to_entries + | map( + .key as $name + | .value as $current + | ($baselineChecks[$name] // null) as $base + | { + key: $name, + value: + if $base == null then + {status:"missing_baseline", currentMs:$current.durationMs} + elif ($current.status != 0) then + {status:"current_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + elif ($base.status != 0) then + {status:"baseline_failed", currentMs:$current.durationMs, baselineMs:$base.durationMs} + else + classify($name; $current.durationMs; $base.durationMs) + end + } + ) + | from_entries + ) as $checks + | ( + if any($checks[]; .status == "fail") then "fail" + elif any($checks[]; .status == "warn") then "warn" + elif any($checks[]; .status == "missing_baseline") then "partial" + else "pass" + end + ) as $status + | {schemaVersion:$schemaVersion, status:$status, mode:$mode, baseline:$baselinePath, checks:$checks} + ' >"$ARTIFACT_DIR/perf-comparison.json" + + local status + status="$(jq -r '.status' "$ARTIFACT_DIR/perf-comparison.json")" + case "$status:$mode" in + fail:fail) + echo "::error::devenv perf regression detected" + jq . "$ARTIFACT_DIR/perf-comparison.json" + return 1 + ;; + fail:*|warn:*) + echo "::warning::devenv perf regression threshold exceeded" + jq . "$ARTIFACT_DIR/perf-comparison.json" + ;; + esac +} + +compare_baseline + if [ -n "${dollar}{GITHUB_STEP_SUMMARY:-}" ]; then { echo "### Devenv perf" @@ -532,6 +714,12 @@ if [ -n "${dollar}{GITHUB_STEP_SUMMARY:-}" ]; then echo "" echo "- Artifact directory: \`$ARTIFACT_DIR\`" echo "- OTEL service: \`${dollar}{OTEL_SERVICE_NAME:-unknown}\`" + echo "" + echo "#### Regression comparison" + echo "" + if [ -f "$ARTIFACT_DIR/perf-comparison.json" ]; then + jq -r '["- Status: " + .status, "- Mode: " + .mode, "- Baseline: " + (.baseline // "none")] | .[]' "$ARTIFACT_DIR/perf-comparison.json" + fi } >>"$GITHUB_STEP_SUMMARY" fi @@ -539,7 +727,9 @@ cat "$ARTIFACT_DIR/timings.pretty.json" ` } -export const devenvPerfBenchmarkStep = (opts?: Pick) => +export const devenvPerfBenchmarkStep = ( + opts?: Pick, +) => ({ name: 'Benchmark devenv surfaces', shell: 'bash', @@ -576,10 +766,17 @@ export const devenvPerfJob = (opts?: DevenvPerfJobOptions) => { ...standardCIEnv, ARTIFACT_DIR: artifactDir, OTEL_SERVICE_NAME: 'devenv-perf-ci', + DEVENV_PERF_REGRESSION_MODE: opts?.regressionMode ?? 'warn', + RUNNER_CLASS: (opts?.runsOn ?? linuxX64Runner).join(','), ...(opts?.env ?? {}), }, steps: [ - ...(opts?.setupSteps ?? [checkoutStep(), installNixStep(), preparePinnedDevenvStep, validateNixStoreStep]), + ...(opts?.setupSteps ?? [ + checkoutStep(), + installNixStep(), + preparePinnedDevenvStep, + validateNixStoreStep, + ]), devenvPerfBenchmarkStep({ taskProbes: opts?.taskProbes, probes: opts?.probes, diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index eb1af80c5..8c30db63a 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -288,6 +288,19 @@ describe('ci workflow devenv perf helpers', () => { it('writes a stable summary artifact for regression tracking', () => { expect(generatedCiWorkflowYamlSource).toContain('schemaVersion: $schemaVersion') expect(generatedCiWorkflowYamlSource).toContain('checks: ($timings[0] | map') + expect(generatedCiWorkflowYamlSource).toContain('measurements.json') + expect(generatedCiWorkflowYamlSource).toContain('--argjson schemaVersion 1') + expect(generatedCiWorkflowYamlSource).toContain('effect-utils-ci-measurement') + expect(generatedCiWorkflowYamlSource).toContain('devenv." + .name + ".duration') + expect(generatedCiWorkflowYamlSource).toContain('target: { kind: "devenv", name: "dev-shell", system: $targetSystem }') + expect(generatedCiWorkflowYamlSource).toContain('RUNNER_CLASS:') + expect(generatedCiWorkflowYamlSource).toContain('namespace-profile-linux-x86-64') + expect(ciWorkflowSource).toContain('nix.closure.nar_size') + expect(ciWorkflowSource).toContain('nix.closure.path_count') + expect(ciWorkflowSource).toContain('nix.closure.bucket.nar_size') + expect(generatedCiWorkflowYamlSource).not.toContain('dev3') + expect(generatedCiWorkflowYamlSource).toContain('perf-comparison.json') + expect(generatedCiWorkflowYamlSource).toContain('DEVENV_PERF_REGRESSION_MODE') expect(generatedCiWorkflowYamlSource).toContain('Upload devenv perf artifacts') expect(generatedCiWorkflowYamlSource).toContain('retention-days: 30') }) From faadaba6cbcdb8262118bb19c1144bba7e44e118 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 13:01:08 +0200 Subject: [PATCH 06/10] Format semantic measurement tests --- .../runtime/github-workflow/ci-workflow-helpers.unit.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index 8c30db63a..765029fb7 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -292,7 +292,9 @@ describe('ci workflow devenv perf helpers', () => { expect(generatedCiWorkflowYamlSource).toContain('--argjson schemaVersion 1') expect(generatedCiWorkflowYamlSource).toContain('effect-utils-ci-measurement') expect(generatedCiWorkflowYamlSource).toContain('devenv." + .name + ".duration') - expect(generatedCiWorkflowYamlSource).toContain('target: { kind: "devenv", name: "dev-shell", system: $targetSystem }') + expect(generatedCiWorkflowYamlSource).toContain( + 'target: { kind: "devenv", name: "dev-shell", system: $targetSystem }', + ) expect(generatedCiWorkflowYamlSource).toContain('RUNNER_CLASS:') expect(generatedCiWorkflowYamlSource).toContain('namespace-profile-linux-x86-64') expect(ciWorkflowSource).toContain('nix.closure.nar_size') From ba5a573e1e5f0eb880bcb78736f6fa7c4626ac39 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 13:08:34 +0200 Subject: [PATCH 07/10] Use portable jq measurement expression --- .github/workflows/ci.yml | 2 +- genie/ci-workflow.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd379bee9..9b9786408 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2411,7 +2411,7 @@ jobs: observations: ( $timings[0] | map({ - name: "devenv." + .name + ".duration", + name: ("devenv." + .name + ".duration"), unit: "seconds", value: (.durationMs / 1000), dimensions: { diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 43fcf02e6..0766fb283 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -578,7 +578,7 @@ jq -n \ observations: ( $timings[0] | map({ - name: "devenv." + .name + ".duration", + name: ("devenv." + .name + ".duration"), unit: "seconds", value: (.durationMs / 1000), dimensions: { From 49fc3f60b3efb07c416032b39ed12daed36d07cd Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 13:39:29 +0200 Subject: [PATCH 08/10] Add Nix closure CI measurements --- genie/ci-workflow.ts | 139 ++++++++++++++++++ .../ci-workflow-helpers.unit.test.ts | 7 + 2 files changed, 146 insertions(+) diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 0766fb283..cf7e26ce4 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -368,6 +368,20 @@ export const ciMeasurementMetrics = { nixClosureBucketNarSize: 'nix.closure.bucket.nar_size', } as const +export type NixClosureMeasurementBucket = { + readonly name: string + readonly pathRegex: string +} + +export type NixClosureMeasurementStepOptions = { + readonly installable: string + readonly targetName?: string + readonly targetSystem?: string + readonly artifactDir?: string + readonly artifactFile?: string + readonly buckets?: readonly NixClosureMeasurementBucket[] +} + type DevenvPerfSetupStep = GitHubWorkflowArgs['jobs'][string]['steps'][number] export type DevenvPerfJobOptions = { @@ -756,6 +770,131 @@ export const devenvPerfArtifactStep = ( }, }) as const +export const nixClosureMeasurementStep = (opts: NixClosureMeasurementStepOptions) => { + const artifactDir = opts.artifactDir ?? 'tmp/ci-measurements' + const artifactFile = opts.artifactFile ?? '$ARTIFACT_DIR/measurements.json' + const targetName = opts.targetName ?? opts.installable + const buckets = JSON.stringify(opts.buckets ?? []) + const targetSystemAssignment = + opts.targetSystem === undefined + ? `target_system="${dollar}{DEVENV_SYSTEM:-${dollar}{RUNNER_OS:-unknown}}"` + : `target_system=${shellSingleQuote(opts.targetSystem)}` + + return { + name: `Measure Nix closure: ${targetName}`, + shell: 'bash', + env: { + ARTIFACT_DIR: artifactDir, + RUNNER_CLASS: '${{ runner.os }}-${{ runner.arch }}', + }, + run: String.raw`set -euo pipefail + +mkdir -p "$ARTIFACT_DIR" +installable=${shellSingleQuote(opts.installable)} +target_name=${shellSingleQuote(targetName)} +artifact_file=${shellSingleQuote(artifactFile)} +${targetSystemAssignment} + +out_path="$(nix build --no-link --print-out-paths "$installable")" +path_info="$ARTIFACT_DIR/nix-closure-path-info.json" +paths_file="$ARTIFACT_DIR/nix-closure-paths.json" + +nix path-info --recursive --json "$out_path" >"$path_info" +jq 'to_entries | map({ path: .key, narSize: (.value.narSize // 0) })' "$path_info" >"$paths_file" + +jq -n \ + --slurpfile paths "$paths_file" \ + --argjson schemaVersion 1 \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "${dollar}{GITHUB_REPOSITORY:-unknown}" \ + --arg branchKind "${dollar}{GITHUB_EVENT_NAME:-unknown}" \ + --arg ref "${dollar}{GITHUB_REF:-unknown}" \ + --arg headSha "${dollar}{GITHUB_SHA:-unknown}" \ + --arg baseSha "${dollar}{GITHUB_BASE_SHA:-}" \ + --arg runnerName "${dollar}{RUNNER_NAME:-unknown}" \ + --arg runnerOs "${dollar}{RUNNER_OS:-unknown}" \ + --arg runnerArch "${dollar}{RUNNER_ARCH:-unknown}" \ + --arg runnerClass "${dollar}{RUNNER_CLASS:-unknown}" \ + --arg githubRunId "${dollar}{GITHUB_RUN_ID:-unknown}" \ + --arg githubRunAttempt "${dollar}{GITHUB_RUN_ATTEMPT:-unknown}" \ + --arg githubJob "${dollar}{GITHUB_JOB:-unknown}" \ + --arg taskId "${dollar}{CROSSTASK_TASK_ID:-}" \ + --arg taskAttemptId "${dollar}{CROSSTASK_ATTEMPT_ID:-}" \ + --arg traceId "${dollar}{TRACE_ID:-}" \ + --arg targetName "$target_name" \ + --arg targetSystem "$target_system" \ + --arg outPath "$out_path" \ + --argjson buckets ${shellSingleQuote(buckets)} \ + ' + ($paths[0] // []) as $closurePaths + | ($closurePaths | map(.narSize) | add // 0) as $totalNarSize + | ($closurePaths | length) as $pathCount + | ($buckets | map( + . as $bucket + | { + name: "nix.closure.bucket.nar_size", + unit: "bytes", + value: ( + $closurePaths + | map(select(.path | test($bucket.pathRegex)) | .narSize) + | add // 0 + ), + dimensions: { bucket: $bucket.name } + } + )) as $bucketObservations + | { + schemaVersion: $schemaVersion, + generatedAt: $generatedAt, + producer: { name: "effect-utils-ci-measurement", version: 1 }, + subject: { + repo: $repository, + branchKind: (if $branchKind == "" then "unknown" else $branchKind end), + ref: $ref, + headSha: $headSha, + baseSha: $baseSha + }, + execution: { + provider: (if ($githubRunId != "" and $githubRunId != "unknown") then "github-actions" else "local" end), + workflow: "CI", + job: $githubJob, + runId: $githubRunId, + runAttempt: $githubRunAttempt, + taskId: $taskId, + attemptId: $taskAttemptId, + traceId: $traceId, + runner: { name: $runnerName, os: $runnerOs, arch: $runnerArch, class: $runnerClass } + }, + target: { kind: "nix-closure", name: $targetName, system: $targetSystem }, + observations: ([ + { + name: "nix.closure.nar_size", + unit: "bytes", + value: $totalNarSize, + dimensions: { bucket: "total" } + }, + { + name: "nix.closure.path_count", + unit: "count", + value: $pathCount, + dimensions: { bucket: "total" } + } + ] + $bucketObservations), + artifacts: [ + { name: "nix-closure-path-info", path: "nix-closure-path-info.json", contentType: "application/json" }, + { name: "nix-closure-paths", path: "nix-closure-paths.json", contentType: "application/json" } + ], + details: { + outPath: $outPath, + topPaths: ($closurePaths | sort_by(.narSize) | reverse | .[:30]) + } + } + ' >"$artifact_file" + +cat "$artifact_file" +`, + } as const +} + export const devenvPerfJob = (opts?: DevenvPerfJobOptions) => { const artifactDir = opts?.artifactDir ?? 'tmp/devenv-perf-ci' diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index 765029fb7..d2b3810bb 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -273,6 +273,8 @@ describe('ci workflow devenv perf helpers', () => { expect(ciWorkflowSource).toContain('export const devenvPerfBenchmarkStep') expect(ciWorkflowSource).toContain('export const devenvPerfArtifactStep') expect(ciWorkflowSource).toContain('export type DevenvPerfProbe') + expect(ciWorkflowSource).toContain('export const nixClosureMeasurementStep') + expect(ciWorkflowSource).toContain('export type NixClosureMeasurementBucket') }) it('emits the standard warm shell and task-list probes with native trace artifacts', () => { @@ -300,6 +302,11 @@ describe('ci workflow devenv perf helpers', () => { expect(ciWorkflowSource).toContain('nix.closure.nar_size') expect(ciWorkflowSource).toContain('nix.closure.path_count') expect(ciWorkflowSource).toContain('nix.closure.bucket.nar_size') + expect(ciWorkflowSource).toContain('target: { kind: "nix-closure"') + expect(ciWorkflowSource).toContain('nix path-info --recursive --json "$out_path"') + expect(ciWorkflowSource).toContain( + 'topPaths: ($closurePaths | sort_by(.narSize) | reverse | .[:30])', + ) expect(generatedCiWorkflowYamlSource).not.toContain('dev3') expect(generatedCiWorkflowYamlSource).toContain('perf-comparison.json') expect(generatedCiWorkflowYamlSource).toContain('DEVENV_PERF_REGRESSION_MODE') From 4ccc6fc0ba95cc662f3b0c9cf6b32ab27221be6a Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 13:49:49 +0200 Subject: [PATCH 09/10] Use current devenv trace flag --- .github/workflows/ci.yml | 3 ++- genie/ci-workflow.ts | 7 +++---- .../github-workflow/ci-workflow-helpers.unit.test.ts | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b9786408..4eb9a21b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2309,6 +2309,7 @@ jobs: case "$arg" in '$DEVENV_BIN') expanded+=("${DEVENV_BIN:?DEVENV_BIN not set}") ;; '$ARTIFACT_DIR'*) expanded+=("${ARTIFACT_DIR}${arg#'$ARTIFACT_DIR'}") ;; + 'json:file:$trace_file') expanded+=("json:file:$trace_file") ;; '$trace_file') expanded+=("file:$trace_file") ;; *) expanded+=("$arg") ;; esac @@ -2328,7 +2329,7 @@ jobs: fi } - measure 'shell_eval_traced' '$ARTIFACT_DIR/traces/shell_eval_traced.json' '$DEVENV_BIN' '--trace-output' '$trace_file' '--trace-format' 'json' 'shell' '--no-reload' '--' 'true' + measure 'shell_eval_traced' '$ARTIFACT_DIR/traces/shell_eval_traced.json' '$DEVENV_BIN' '--trace-to' 'json:file:$trace_file' 'shell' '--no-reload' '--' 'true' measure 'shell_eval_warm' '' '$DEVENV_BIN' 'shell' '--no-reload' '--' 'true' measure 'tasks_list' '' '$DEVENV_BIN' 'tasks' 'list' measure 'processes_help' '' '$DEVENV_BIN' 'processes' '--help' diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index cf7e26ce4..9dfadbba8 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -415,10 +415,8 @@ const renderDevenvPerfScript = ( name: 'shell_eval_traced', command: [ '$DEVENV_BIN', - '--trace-output', - '$trace_file', - '--trace-format', - 'json', + '--trace-to', + 'json:file:$trace_file', 'shell', '--no-reload', '--', @@ -496,6 +494,7 @@ measure() { case "$arg" in '$DEVENV_BIN') expanded+=("${dollar}{DEVENV_BIN:?DEVENV_BIN not set}") ;; '$ARTIFACT_DIR'*) expanded+=("${dollar}{ARTIFACT_DIR}${dollar}{arg#'$ARTIFACT_DIR'}") ;; + 'json:file:$trace_file') expanded+=("json:file:$trace_file") ;; '$trace_file') expanded+=("file:$trace_file") ;; *) expanded+=("$arg") ;; esac diff --git a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts index d2b3810bb..126dca2bf 100644 --- a/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts +++ b/packages/@overeng/genie/src/runtime/github-workflow/ci-workflow-helpers.unit.test.ts @@ -281,7 +281,8 @@ describe('ci workflow devenv perf helpers', () => { expect(generatedCiWorkflowYamlSource).toContain('devenv-perf:') expect(generatedCiWorkflowYamlSource).toContain('OTEL_SERVICE_NAME: devenv-perf-ci') expect(generatedCiWorkflowYamlSource).toContain("measure 'shell_eval_traced'") - expect(generatedCiWorkflowYamlSource).toContain('--trace-output') + expect(generatedCiWorkflowYamlSource).toContain('--trace-to') + expect(generatedCiWorkflowYamlSource).toContain('json:file:$trace_file') expect(generatedCiWorkflowYamlSource).toContain('$ARTIFACT_DIR/traces/shell_eval_traced.json') expect(generatedCiWorkflowYamlSource).toContain("measure 'shell_eval_warm'") expect(generatedCiWorkflowYamlSource).toContain("measure 'tasks_list'") From c2edd44ffdf8da7fef8b9c65aecd9c3b70760a40 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Sun, 10 May 2026 14:00:38 +0200 Subject: [PATCH 10/10] Expand trace artifact path before running devenv --- .github/workflows/ci.yml | 3 +++ genie/ci-workflow.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4eb9a21b4..6e53ab2c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2297,6 +2297,9 @@ jobs: local name="$1" local trace_file="$2" shift 2 + case "$trace_file" in + '$ARTIFACT_DIR'*) trace_file="${ARTIFACT_DIR}${trace_file#'$ARTIFACT_DIR'}" ;; + esac local stdout="$ARTIFACT_DIR/$name.stdout" local stderr="$ARTIFACT_DIR/$name.stderr" local started ended status duration_ms diff --git a/genie/ci-workflow.ts b/genie/ci-workflow.ts index 9dfadbba8..2d4feb885 100644 --- a/genie/ci-workflow.ts +++ b/genie/ci-workflow.ts @@ -482,6 +482,9 @@ measure() { local name="$1" local trace_file="$2" shift 2 + case "$trace_file" in + '$ARTIFACT_DIR'*) trace_file="${dollar}{ARTIFACT_DIR}${dollar}{trace_file#'$ARTIFACT_DIR'}" ;; + esac local stdout="$ARTIFACT_DIR/$name.stdout" local stderr="$ARTIFACT_DIR/$name.stderr" local started ended status duration_ms