zig-js/.github/workflows/performance.yml at main · zig-utils/zig-js · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
name: Performance

on:
  workflow_dispatch:
    inputs:
      samples:
        description: Full-matrix samples per row
        required: true
        default: '7'
        type: string
      lanes:
        description: Comma-separated lane counts above the implicit one-lane baseline
        required: true
        default: '2,4,8'
        type: string
      baseline_raw:
        description: Optional committed raw TSV to compare on a matching controlled runner
        required: false
        default: ''
        type: string
      baseline_report:
        description: Optional committed Markdown report paired with baseline_raw
        required: false
        default: ''
        type: string

permissions:
  contents: read

concurrency:
  group: benchmark-comparison-${{ github.ref }}
  cancel-in-progress: false

jobs:
  comparison:
    # Manual evidence collection only: hosted-runner timings are artifacts for
    # inspection and are never an ordinary-CI regression gate. Promotion into
    # docs/README additionally requires the driver's clean-tree checks and a
    # like-for-like environment review against the retained reference report.
    runs-on: macos-15
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
        with:
          path: zig-js
      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
        with:
          repository: zig-utils/zig-regex
          path: zig-regex
          ref: 2de46683b948ec895e5fa9a9e7e4c384aceccdfe
      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
        with:
          repository: zig-utils/zig-gc
          path: zig-gc
          ref: 98edfaa4aa1b2676680d95801d14118f8ed84156
      # benchmark-comparison-test includes the representative-matrix contract,
      # which is a durable TypeScript tool executed by the owned Home runtime.
      # Keep this producer chain identical to the main CI job: zig-js first
      # supplies the public C archive, then the pinned Home checkout links its
      # repository-tool runner against that archive.
      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
        with:
          repository: home-lang/home
          path: Home/lang
          ref: d01469e9dd90e84cf9434b056390dcb4667b18bc
      - uses: pantry-pm/pantry/packages/action@4ad0cc7b728ba6dc295fcccf9b88240f9fe610c9
        with:
          packages: zig@0.17.0-dev.1441+d5181a9c9

      - name: Build zig-js public C API for Home
        working-directory: zig-js
        run: zig build home-tool-bootstrap

      - name: Build pinned Home repository-tool runtime
        working-directory: Home/lang
        run: zig build home-tool -Dtool-js-engine=zig-js -Dzig-js-root="$GITHUB_WORKSPACE/zig-js"

      - name: Validate comparison harness
        working-directory: zig-js
        run: zig build benchmark-comparison-test -Dhome-tool="$GITHUB_WORKSPACE/Home/lang/zig-out/bin/home-tool"

      - name: Run full symmetric matrix
        working-directory: zig-js
        env:
          BENCHMARK_SAMPLES: ${{ inputs.samples }}
          BENCHMARK_LANES: ${{ inputs.lanes }}
        run: |
          mkdir -p benchmark-results
          zig build benchmark-comparison \
            -Dbenchmark-comparison-samples="$BENCHMARK_SAMPLES" \
            -Dbenchmark-comparison-lanes="$BENCHMARK_LANES" \
            -Dbenchmark-comparison-raw-out="benchmark-results/benchmark-comparison-${GITHUB_RUN_ID}.tsv" \
            -Dbenchmark-comparison-markdown-out="benchmark-results/benchmark-comparison-${GITHUB_RUN_ID}.md"

      - name: Validate publication and generate scorecard artifact
        working-directory: zig-js
        run: |
          cp README.md benchmark-results/README.generated.md
          "$GITHUB_WORKSPACE/Home/lang/zig-out/bin/home-tool" run tools/benchmark-publication.ts \
            --current-raw="benchmark-results/benchmark-comparison-${GITHUB_RUN_ID}.tsv" \
            --current-report="benchmark-results/benchmark-comparison-${GITHUB_RUN_ID}.md" \
            --readme=benchmark-results/README.generated.md

      - name: Compare controlled historical baseline
        if: inputs.baseline_raw != '' || inputs.baseline_report != ''
        working-directory: zig-js
        env:
          BASELINE_RAW: ${{ inputs.baseline_raw }}
          BASELINE_REPORT: ${{ inputs.baseline_report }}
        run: |
          "$GITHUB_WORKSPACE/Home/lang/zig-out/bin/home-tool" run tools/benchmark-publication.ts \
            --current-raw="benchmark-results/benchmark-comparison-${GITHUB_RUN_ID}.tsv" \
            --current-report="benchmark-results/benchmark-comparison-${GITHUB_RUN_ID}.md" \
            --baseline-raw="$BASELINE_RAW" \
            --baseline-report="$BASELINE_REPORT" \
            --history-out="benchmark-results/benchmark-history-${GITHUB_RUN_ID}.md"

      - name: Upload raw evidence and report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: benchmark-comparison-${{ github.run_id }}
          path: zig-js/benchmark-results/
          if-no-files-found: error
          retention-days: 90