From 72ac5d897892c5b672abf51d5e4a71c6508eb4d3 Mon Sep 17 00:00:00 2001 From: Luyu Zhang Date: Sun, 26 Apr 2026 11:50:06 -0700 Subject: [PATCH 1/5] chore(ci): add macos runner benchmark --- .github/workflows/runner-benchmark.yml | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/runner-benchmark.yml diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml new file mode 100644 index 000000000..a5fc1c322 --- /dev/null +++ b/.github/workflows/runner-benchmark.yml @@ -0,0 +1,52 @@ +name: Runner Benchmark + +on: + pull_request: + branches: + - main + workflow_dispatch: + +concurrency: + group: runner-benchmark-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + include: + - runner: macos-latest + name: github-macos + - runner: depot-macos-latest + name: depot-macos + runs-on: ${{ matrix.runner }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.23" + + - name: Benchmark CLI build + id: bench + shell: bash + run: | + start=$(date +%s) + go mod download + CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o /tmp/dify-plugin-darwin-amd64 ./cmd/commandline + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o /tmp/dify-plugin-darwin-arm64 ./cmd/commandline + end=$(date +%s) + echo "elapsed=$((end - start))" >> "$GITHUB_OUTPUT" + + - name: Summarize + shell: bash + run: | + { + echo "### Runner Benchmark" + echo "" + echo "- Runner: ${{ matrix.name }}" + echo "- Elapsed seconds: ${{ steps.bench.outputs.elapsed }}" + } >> "$GITHUB_STEP_SUMMARY" From 4e9052f72c5c154d5294d3d96c29e4383e5a1050 Mon Sep 17 00:00:00 2001 From: Luyu Zhang Date: Sun, 26 Apr 2026 11:56:35 -0700 Subject: [PATCH 2/5] chore(ci): benchmark cold and warm macos cache --- .github/workflows/runner-benchmark.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index a5fc1c322..f72a0fe47 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -31,7 +31,21 @@ jobs: go-version: "1.23" - name: Benchmark CLI build - id: bench + id: bench_cold + shell: bash + run: | + rm -rf "$GOMODCACHE" "$GOCACHE" + mkdir -p "$GOMODCACHE" "$GOCACHE" + + start=$(date +%s) + go mod download + CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o /tmp/dify-plugin-darwin-amd64 ./cmd/commandline + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o /tmp/dify-plugin-darwin-arm64 ./cmd/commandline + end=$(date +%s) + echo "elapsed=$((end - start))" >> "$GITHUB_OUTPUT" + + - name: Benchmark CLI build warm cache + id: bench_warm shell: bash run: | start=$(date +%s) @@ -48,5 +62,6 @@ jobs: echo "### Runner Benchmark" echo "" echo "- Runner: ${{ matrix.name }}" - echo "- Elapsed seconds: ${{ steps.bench.outputs.elapsed }}" + echo "- Cold cache seconds: ${{ steps.bench_cold.outputs.elapsed }}" + echo "- Warm cache seconds: ${{ steps.bench_warm.outputs.elapsed }}" } >> "$GITHUB_STEP_SUMMARY" From 579f6788031f904ccb7bf987f0603ebdc2e823f2 Mon Sep 17 00:00:00 2001 From: Luyu Zhang Date: Sun, 26 Apr 2026 12:03:02 -0700 Subject: [PATCH 3/5] fix(ci): resolve benchmark cache paths --- .github/workflows/runner-benchmark.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index f72a0fe47..232b11509 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -34,8 +34,10 @@ jobs: id: bench_cold shell: bash run: | - rm -rf "$GOMODCACHE" "$GOCACHE" - mkdir -p "$GOMODCACHE" "$GOCACHE" + gomodcache=$(go env GOMODCACHE) + gocache=$(go env GOCACHE) + rm -rf "$gomodcache" "$gocache" + mkdir -p "$gomodcache" "$gocache" start=$(date +%s) go mod download From a9627922643af0c4af54324f19eae97a95a4a07c Mon Sep 17 00:00:00 2001 From: Luyu Zhang Date: Sun, 26 Apr 2026 12:07:27 -0700 Subject: [PATCH 4/5] fix(ci): isolate benchmark cache dirs --- .github/workflows/runner-benchmark.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index 232b11509..47016c037 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -12,6 +12,8 @@ concurrency: jobs: benchmark: + env: + BENCH_CACHE_BASE: ${{ runner.temp }}/runner-benchmark strategy: fail-fast: false matrix: @@ -33,11 +35,12 @@ jobs: - name: Benchmark CLI build id: bench_cold shell: bash + env: + GOMODCACHE: ${{ runner.temp }}/runner-benchmark/cold/gomod + GOCACHE: ${{ runner.temp }}/runner-benchmark/cold/gocache run: | - gomodcache=$(go env GOMODCACHE) - gocache=$(go env GOCACHE) - rm -rf "$gomodcache" "$gocache" - mkdir -p "$gomodcache" "$gocache" + rm -rf "$BENCH_CACHE_BASE/cold" + mkdir -p "$GOMODCACHE" "$GOCACHE" start=$(date +%s) go mod download @@ -49,6 +52,9 @@ jobs: - name: Benchmark CLI build warm cache id: bench_warm shell: bash + env: + GOMODCACHE: ${{ runner.temp }}/runner-benchmark/cold/gomod + GOCACHE: ${{ runner.temp }}/runner-benchmark/cold/gocache run: | start=$(date +%s) go mod download From 1fb57ee52ff89187651cf6b2750bcb65cd9bd5c4 Mon Sep 17 00:00:00 2001 From: Luyu Zhang Date: Sun, 26 Apr 2026 12:14:13 -0700 Subject: [PATCH 5/5] fix(ci): use runner temp in benchmark cache paths --- .github/workflows/runner-benchmark.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index 47016c037..d6aa435d3 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -12,8 +12,6 @@ concurrency: jobs: benchmark: - env: - BENCH_CACHE_BASE: ${{ runner.temp }}/runner-benchmark strategy: fail-fast: false matrix: @@ -35,11 +33,11 @@ jobs: - name: Benchmark CLI build id: bench_cold shell: bash - env: - GOMODCACHE: ${{ runner.temp }}/runner-benchmark/cold/gomod - GOCACHE: ${{ runner.temp }}/runner-benchmark/cold/gocache run: | - rm -rf "$BENCH_CACHE_BASE/cold" + cache_base="$RUNNER_TEMP/runner-benchmark/cold" + export GOMODCACHE="$cache_base/gomod" + export GOCACHE="$cache_base/gocache" + rm -rf "$cache_base" mkdir -p "$GOMODCACHE" "$GOCACHE" start=$(date +%s) @@ -52,10 +50,10 @@ jobs: - name: Benchmark CLI build warm cache id: bench_warm shell: bash - env: - GOMODCACHE: ${{ runner.temp }}/runner-benchmark/cold/gomod - GOCACHE: ${{ runner.temp }}/runner-benchmark/cold/gocache run: | + cache_base="$RUNNER_TEMP/runner-benchmark/cold" + export GOMODCACHE="$cache_base/gomod" + export GOCACHE="$cache_base/gocache" start=$(date +%s) go mod download CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o /tmp/dify-plugin-darwin-amd64 ./cmd/commandline